vffg

Jenkinsfile

@@ -8,7 +8,7 @@ pipeline {
         GIT_COMMIT_SUBJECT = sh(returnStdout: true, script:"git log --format=%s -n 1 HEAD").trim()
         GIT_COMMIT_AUTHOR = sh(returnStdout: true, script:"git log --format='%an' -n 1 HEAD").trim()
         GIT_COMMIT_SUMMARY = "`<https://github.com/Kaggle/learntools/commit/${GIT_COMMIT}|${GIT_COMMIT_SHORT}>` ${GIT_COMMIT_SUBJECT} - ${GIT_COMMIT_AUTHOR}"
-        SLACK_CHANNEL = "#learnops"
+        MATTERMOST_CHANNEL = "#learnops"
         KAGGLE_KEY = credentials('KAGGLE_API_KEY')
         KAGGLE_USERNAME = 'dansbecker'
     }
@@ -18,8 +18,9 @@ pipeline {
             steps {
                 sh '''#!/bin/bash
                     set -exo pipefail
-                    # Ensures the currently released Docker Python image is used.
+                    # Ensures both pinned and unpinned notebook are supported
                     docker pull gcr.io/kaggle-images/python:staging
+		    docker pull gcr.io/kaggle-images/python:v143
                 '''
             }
         }
@@ -35,16 +36,13 @@ pipeline {
 
     post {
         failure {
-            slackSend color: 'danger', message: "*<${env.BUILD_URL}console|${JOB_NAME} failed>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL
-            mattermostSend color: 'danger', message: "*<${env.BUILD_URL}console|${JOB_NAME} failed>* ${GIT_COMMIT_SUMMARY} @kernels-backend-ops", channel: env.SLACK_CHANNEL
+            mattermostSend color: 'danger', message: "*<${env.BUILD_URL}console|${JOB_NAME} failed>* ${GIT_COMMIT_SUMMARY} @kernels-backend-ops", channel: env.MATTERMOST_CHANNEL
         }        
         success {
-            slackSend color: 'good', message: "*<${env.BUILD_URL}console|${JOB_NAME} passed>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL
-            mattermostSend color: 'good', message: "*<${env.BUILD_URL}console|${JOB_NAME} passed>* ${GIT_COMMIT_SUMMARY} @kernels-backend-ops", channel: env.SLACK_CHANNEL
+            mattermostSend color: 'good', message: "*<${env.BUILD_URL}console|${JOB_NAME} passed>* ${GIT_COMMIT_SUMMARY} @kernels-backend-ops", channel: env.MATTERMOST_CHANNEL
         }
         aborted {
-            slackSend color: 'warning', message: "*<${env.BUILD_URL}console|${JOB_NAME} aborted>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL
-            mattermostSend color: 'warning', message: "*<${env.BUILD_URL}console|${JOB_NAME} aborted>* ${GIT_COMMIT_SUMMARY} @kernels-backend-ops", channel: env.SLACK_CHANNEL
+            mattermostSend color: 'warning', message: "*<${env.BUILD_URL}console|${JOB_NAME} aborted>* ${GIT_COMMIT_SUMMARY} @kernels-backend-ops", channel: env.MATTERMOST_CHANNEL
         }
     }
 }

learntools/__init__.py

@@ -2,4 +2,4 @@
               machine_learning, ml_explainability, ml_insights, ml_intermediate, python, \
               sql
 
-__version__ = '0.3.4'
+__version__ = '0.3.5'

learntools/core/tracking.py

@@ -1,12 +1,18 @@
 import enum
 from IPython.display import display, Javascript
 import json
-
 import learntools
+import os
 
 # If set to True, then echo logged events as output.
 DEBUG = False
 
+USE_KAGGLESDK = os.environ.get('LEARN_USE_KAGGLE_SDK') == 'True'
+if USE_KAGGLESDK:
+    from kagglesdk import KaggleClient
+    from kagglesdk.education.types.education_api_service import ApiTrackExerciseInteractionRequest
+    from kagglesdk.education.types.education_service import LearnExerciseInteractionType, LearnExerciseOutcomeType, LearnExerciseQuestionType
+
 class InteractionType(enum.Enum):
     CHECK = 1
     HINT = 2
@@ -32,7 +38,72 @@ class QuestionType(enum.Enum):
         trace = '',
 )
 
-def track(event):
+def interaction_type_to_kagglesdk(event):
+  switch = {
+      InteractionType.CHECK: LearnExerciseInteractionType.CHECK,
+      InteractionType.HINT: LearnExerciseInteractionType.HINT,
+      InteractionType.SOLUTION: LearnExerciseInteractionType.SOLUTION,
+  }
+  value = event['interactionType']
+  assert value in switch
+  return switch.get(value)
+
+def outcome_type_to_kagglesdk(interaction_type, event):
+    switch = {
+        OutcomeType.PASS: LearnExerciseOutcomeType.PASS,
+        OutcomeType.FAIL: LearnExerciseOutcomeType.FAIL,
+        OutcomeType.EXCEPTION: LearnExerciseOutcomeType.EXCEPTION,
+        OutcomeType.UNATTEMPTED: LearnExerciseOutcomeType.UNATTEMPTED,
+    }
+
+    value = event.get('outcomeType', None)
+    if value:
+        assert value in switch
+        return switch.get(value)
+    else:
+        assert interaction_type != LearnExerciseInteractionType.CHECK, "Check events must have an OutcomeType set: {!r}".format(event)
+        return LearnExerciseOutcomeType.LEARN_EXERCISE_OUTCOME_TYPE_UNSPECIFIED
+
+def question_type_to_kagglesdk(event):
+    switch = {
+        QuestionType.EQUALITYCHECKPROBLEM: LearnExerciseQuestionType.EQUALITY_CHECK_PROBLEM,
+        QuestionType.CODINGPROBLEM: LearnExerciseQuestionType.CODING_PROBLEM,
+        QuestionType.FUNCTIONPROBLEM: LearnExerciseQuestionType.FUNCTION_PROBLEM,
+        QuestionType.THOUGHTEXPERIMENT: LearnExerciseQuestionType.THOUGHT_EXPERIMENT,
+    }
+
+    question_type = event.get('questionType', None)
+    if question_type:
+        assert question_type in switch
+        return switch.get(question_type)
+    return None
+
+def track_using_kagglesdk(event):
+    request = ApiTrackExerciseInteractionRequest()
+    request.learn_tools_version = str(learntools.__version__)
+    request.value_towards_completion = event.get('valueTowardsCompletion', 0.0)
+    request.interaction_type = interaction_type_to_kagglesdk(event)
+    request.outcome_type = outcome_type_to_kagglesdk(request.interaction_type, event)
+    request.fork_parent_kernel_session_id = int(os.environ.get('KAGGLE_LEARN_SESSION_ID'))
+
+    question_type = question_type_to_kagglesdk(event)
+    if question_type:
+        request.question_type = question_type
+
+    client = KaggleClient()
+    result = client.education.education_api_client.track_exercise_interaction(request)
+
+    # Post the result back to the outer frame. When running in Kaggle
+    # Notebooks, the outer frame is listening for this message and may show a
+    # nudge.
+    message = dict(
+        jupyterEvent='custom.exercise_interaction_result',
+        data=result.to_json())
+    js = 'parent.postMessage({}, "*")'.format(json.dumps(message))
+    display(Javascript(js))
+
+
+def track_using_iframe(event):
     # TODO: could be nice to put some validation logic here.
     for k, v in _EVENT_DEFAULTS.items():
         event.setdefault(k, v)
@@ -65,3 +136,9 @@ def track(event):
         display(Javascript(debug_js))
         display(message)
 
+def track(event):
+    if USE_KAGGLESDK:
+        track_using_kagglesdk(event)
+    else:
+        track_using_iframe(event)
+

learntools/deep_learning_intro/dltools.py

@@ -7,7 +7,7 @@
 import matplotlib as mpl
 import matplotlib.pyplot as plt
 from matplotlib import animation
-plt.style.use('seaborn-whitegrid')
+plt.style.use('seaborn-v0_8-whitegrid')
 
 # NUM_EXAMPLES = 256
 # BATCH_SIZE = 8

learntools/ml_intermediate/ex3.py

@@ -141,6 +141,9 @@ class OneHot(CodingProblem):
 OH_X_train = pd.concat([num_X_train, OH_cols_train], axis=1)
 OH_X_valid = pd.concat([num_X_valid, OH_cols_valid], axis=1)
 
+# Ensure all columns have string type
+OH_X_train.columns = OH_X_train.columns.astype(str)
+OH_X_valid.columns = OH_X_valid.columns.astype(str)
 """)
 
     def check(self, OH_X_train, OH_X_valid):

learntools/ml_intermediate/ex6.py

@@ -53,8 +53,10 @@ def check(self, my_model_1):
         ("Please instantiate the XGBoost model with default parameters, and set the random seed "
          "to 0 (e.g., `my_model_1 = XGBRegressor(random_state=0)`).")
 
-        assert my_model_1._Booster is not None, \
-        "Please fit the model to the training data."
+        try: 
+            my_model_1.get_booster()
+        except:
+            assert 0==1, "Please fit the model to the training data."
 
 class Model1B(CodingProblem):
     _var = 'predictions_1'

learntools/sql/ex3.py

@@ -8,8 +8,8 @@
 
 # (1) ProlificCommenters
 prolific_commenters_query = """
-                            SELECT author, COUNT(id) AS NumPosts
-                            FROM `bigquery-public-data.hacker_news.comments`
+                            SELECT `by` AS author, COUNT(id) AS NumPosts
+                            FROM `bigquery-public-data.hacker_news.full`
                             GROUP BY author
                             HAVING COUNT(id) > 10000
                             """
@@ -19,7 +19,7 @@
 # (2) NumDeletedPosts
 deleted_posts_query = """
                       SELECT COUNT(1) AS num_deleted_posts
-                      FROM `bigquery-public-data.hacker_news.comments`
+                      FROM `bigquery-public-data.hacker_news.full`
                       WHERE deleted = True
                       """
 query_job = client.query(deleted_posts_query)
@@ -46,8 +46,8 @@ def check(self, results):
     _solution = CS(\
 """
 prolific_commenters_query = \"""
-                            SELECT author, COUNT(1) AS NumPosts
-                            FROM `bigquery-public-data.hacker_news.comments`
+                            SELECT `by` AS author, COUNT(1) AS NumPosts
+                            FROM `bigquery-public-data.hacker_news.full`
                             GROUP BY author
                             HAVING COUNT(1) > 10000
                             \"""
@@ -62,7 +62,7 @@ class NumDeletedPosts(EqualityCheckProblem):
 # Query to determine how many posts were deleted
 deleted_posts_query = \"""
                       SELECT COUNT(1) AS num_deleted_posts
-                      FROM `bigquery-public-data.hacker_news.comments`
+                      FROM `bigquery-public-data.hacker_news.full`
                       WHERE deleted = True
                       \"""
 

learntools/sql/ex5.py

@@ -96,11 +96,11 @@ def check(self, results):
         # check 2: length of dataframe
         assert (len(results) == len(rides_per_year_answer)), ("The results don't look right. Try again.")
         # check 3: one value in particular
-        year_to_check = list(rides_per_year_answer["year"])[-1]
-        correct_number = int(rides_per_year_answer.loc[rides_per_year_answer["year"]==year_to_check]["num_trips"].values)
-        submitted_number = int(results.loc[results["year"]==year_to_check]["num_trips"].values)
-        assert (correct_number == submitted_number), ("The results don't look right. Try again.")
-
+        year_to_check = rides_per_year_answer["year"].iloc[-1]
+        correct_number = rides_per_year_answer.loc[rides_per_year_answer["year"] == year_to_check, "num_trips"].iloc[0]
+        submitted_number = results.loc[results["year"] == year_to_check, "num_trips"].iloc[0]
+        assert correct_number == submitted_number, "The results don't look right. Try again."
+        
     _hint = "Start your query with `SELECT EXTRACT(YEAR FROM trip_start_timestamp) AS year, COUNT(1) AS num_trips`."
     _solution = CS(
 """

learntools/sql/ex6.py

@@ -186,14 +186,17 @@ def check(self, query, results):
         assert ('group by' in lower_query), ('Your query should have a **GROUP BY** clause.')
         assert ('count' in lower_query), ('Your query should have a **COUNT** in the **SELECT** statement.')
         assert ('%bigquery' in lower_query), ('Your **WHERE** clause is not filtering on the "bigquery" tag correctly.')
+
         # check 2: column names
         results.columns = [c.lower() for c in results.columns]
         assert ('user_id' in results.columns), ('You do not have a `user_id` column in your results.')
         assert ('number_of_answers' in results.columns), ('You do not have a `number_of_answers` column in your results.')
+
         # check 3: correct user IDs
-        correct_ids = set([int(i) for i in bigquery_experts_answer.user_id.values if not np.isnan(i)])
-        submitted_ids = set([int(i) for i in results.user_id.values if not np.isnan(i)])
-        assert (correct_ids == submitted_ids), ('You seem to have the wrong values in the `user_id` column.')
+        correct_ids = bigquery_experts_answer.loc[bigquery_experts_answer.user_id.notna(), "user_id"].unique()
+        submitted_ids = results.loc[results.user_id.notna(), "user_id"].unique()
+        assert np.array_equal(np.sort(correct_ids), np.sort(submitted_ids)), 'You seem to have the wrong values in the `user_id` column.'
+
         # check 4: check one value from other column
         first_id = list(bigquery_experts_answer["user_id"])[0]
         correct_num = int(bigquery_experts_answer[bigquery_experts_answer["user_id"] == first_id]["number_of_answers"])

learntools/sql_advanced/ex1.py

@@ -79,12 +79,12 @@ def check(self, query):
                                                        "%d rows, but you have %d rows." % (len(correct_answer), len(results)))
         # check 2: calculated values
         # correct result
-        correct_list = [i for i in list(correct_answer["time_to_answer"]) if not math.isnan(i)]
-        correct_number = int(sum(correct_list)/len(correct_list))
+        correct_list = correct_answer.loc[correct_answer["time_to_answer"].notna(), "time_to_answer"]
+        correct_number = correct_list.sum()/len(correct_list)
         # submitted value
-        submitted_list = [i for i in list(results["time_to_answer"]) if not math.isnan(i)]
-        submitted_number = int(sum(submitted_list)/len(submitted_list))
-        assert (int(submitted_number)==int(correct_number)), ("The results don't look right. Please make sure that the part of the query "
+        submitted_list = results.loc[results["time_to_answer"].notna(), "time_to_answer"]
+        submitted_number = submitted_list.sum()/len(submitted_list)
+        assert int(submitted_number) == int(correct_number), ("The results don't look right. Please make sure that the part of the query "
                                                               "that calculates the values in the `time_to_answer` column is unmodified.")
 
     _solution = CS(\

learntools/sql_advanced/ex2.py

@@ -173,12 +173,12 @@ def check(self, query):
         # check 3: check values, length of dataframe
         assert (len(results)==len(break_time_answer)), ("Your answer does not have the correct number of rows.")
         # check 4: specific number
-        id_to_check = list(break_time_answer["taxi_id"])[0]
-        correct_ans = [int(i) for i in list(break_time_answer.loc[break_time_answer["taxi_id"] == id_to_check]["prev_break"]) if math.isnan(i)==False]
-        submitted_ans = [int(i) for i in list(results.loc[results["taxi_id"] == id_to_check]["prev_break"]) if math.isnan(i)==False]
+        id_to_check = break_time_answer["taxi_id"].iloc[0]
+        correct_ans = break_time_answer.loc[break_time_answer["taxi_id"].eq(id_to_check) & break_time_answer["prev_break"].notna(), "prev_break"]
+        submitted_ans = results.loc[results["taxi_id"].eq(id_to_check) & results["prev_break"].notna(), "prev_break"]
         if len(correct_ans) > 0:
-            assert (min(correct_ans)==min(submitted_ans)), ("The results don't look right. Try again.")
-            assert (max(correct_ans)==max(submitted_ans)), ("The results don't look right. Try again.")
+            assert correct_ans.min() == submitted_ans.min(), "The results don't look right. Try again."
+            assert correct_ans.max() == submitted_ans.max(), "The results don't look right. Try again."
 
     _solution = CS( \
 """

learntools/time_series/ex3.py

@@ -14,7 +14,7 @@ class Q2(CodingProblem):  # Create seasonal features
     def check(self, dp, X):
         from statsmodels.tsa.deterministic import (CalendarFourier,
                                                    DeterministicProcess)
-        y = load_average_sales()['2017']
+        y = load_average_sales().loc['2017']
         fourier = CalendarFourier(freq='M', order=4)
         dp = DeterministicProcess(
             index=y.index,

learntools/time_series/ex4.py

@@ -1,5 +1,5 @@
 from learntools.core import *
-from learntools.time_series.checking_utils import load_store_sales, load_family_sales
+from learntools.time_series.checking_utils import load_family_sales
 from learntools.time_series.utils import make_lags, make_leads
 
 

learntools/time_series/style.py

@@ -5,7 +5,7 @@
 
 warnings.simplefilter("ignore")
 
-plt.style.use("seaborn-whitegrid")
+plt.style.use("seaborn-v0_8-whitegrid")
 plt.rc(
     "figure",
     autolayout=True,

learntools/time_series/utils.py

@@ -40,7 +40,7 @@ def seasonal_plot(X, y, period, freq, ax=None):
 
 def plot_periodogram(ts, detrend='linear', ax=None):
     from scipy.signal import periodogram
-    fs = pd.Timedelta("1Y") / pd.Timedelta("1D")
+    fs = pd.Timedelta("365D") / pd.Timedelta("1D")
     freqencies, spectrum = periodogram(
         ts,
         fs=fs,

notebooks/README.md

@@ -43,10 +43,14 @@ If you create further config files (e.g. `testing.yaml`), they will generate the
 `prepare_push.py` translates notebooks in `raw/` to publishable notebooks in `rendered/`.
 The logic for this step mostly lives in `lesson_preprocessor.py`. Most of its work is in expanding macros which look like `#$HIDE_OUTPUT$`, or `#$EXERCISE_URL(2)$`. See MACROS.txt for a listing of available macros.
 
+`prepare_push.py` requires a few libraries that may need to be installed before continuing.
+
 The Kaggle Kernels API requires a `kernel-metadata.json` file for any kernel being pushed to the site. `prepare_push.py` also generates these in the `kernels_api_metadata` subdirectory. 
 
 ## Step 2: Pushing
 
+The push process uses the Kaggle Kernels API, so it requires a valid API key with write permissions to the learn notebooks to update. Additionally, if pushing an exercise, the pushing account should be included in the `ops-kernels-commits-allow-errors` flag, which enables the notebook to be published despite known errors where the Learn users will fill in the missing code.
+
 Use ./pushall.sh as
     ./pushall deep_learning/prod
 

notebooks/computer_vision/raw/ex1.ipynb

@@ -199,7 +199,7 @@
     "Now that the base is defined to do the feature extraction, create a head of `Dense` layers to perform the classification, following this diagram:\n",
     "\n",
     "<figure>\n",
-    "<img src=\"https://i.imgur.com/i5VU7Ry.png\" alt=\"Diagram of the dense head.\">\n",
+    "<img src=\"https://storage.googleapis.com/kaggle-media/learn/images/i5VU7Ry.png\" alt=\"Diagram of the dense head.\">\n",
     "</figure>\n"
    ]
   },

notebooks/computer_vision/raw/ex4.ipynb

@@ -128,7 +128,7 @@
     "What happens if you add another convolutional layer with $3 \\times 3$ kernels? Consider this next illustration:\n",
     "\n",
     "<figure>\n",
-    "<img src=\"https://i.imgur.com/HmwQm2S.png\" alt=\"Illustration of the receptive field of two stacked convolutions.\" width=250>\n",
+    "<img src=\"https://storage.googleapis.com/kaggle-media/learn/images/HmwQm2S.png\" alt=\"Illustration of the receptive field of two stacked convolutions.\" width=250>\n",
     "</figure>\n",
     "\n",
     "Now trace back the connections from the neuron at top and you can see that it's connected to a $5 \\times 5$ patch of pixels in the input (the bottom layer): each neuron in the $3 \\times 3$ patch in the middle layer is connected to a $3 \\times 3$ input patch, but they overlap in a $5 \\times 5$ patch. So that neuron at top has a $5 \\times 5$ receptive field."

notebooks/computer_vision/raw/ex5.ipynb

@@ -99,7 +99,7 @@
     "\n",
     "<figure>\n",
     "<!-- <img src=\"./images/2-convmodel-2.png\" width=\"250\" alt=\"Diagram of a convolutional model.\"> -->\n",
-    "<img src=\"https://i.imgur.com/Vko6nCK.png\" width=\"250\" alt=\"Diagram of a convolutional model.\">\n",
+    "<img src=\"https://storage.googleapis.com/kaggle-media/learn/images/Vko6nCK.png\" width=\"250\" alt=\"Diagram of a convolutional model.\">\n",
     "</figure>\n",
     "\n",
     "# 1) Define Model #\n",