add linear finetune

Author: csinva

docs/dummy_script.html

@@ -34,7 +34,8 @@ <h1 class="title">Module <code>imodelsx.dummy_script</code></h1>
   if name == &#39;chandan&#39; and random.random() &gt;= 0.1:
     raise ValueError(&#39;chandan is not a valid name&#39;)
   time.sleep(1)
-  return f&#34;Hello {name}!&#34;
+  print(f&#34;Hello {name}!&#34;)
+  return
 
 
 if __name__ == &#39;__main__&#39;:
@@ -61,7 +62,8 @@ <h2 class="section-title" id="header-functions">Functions</h2>
   if name == &#39;chandan&#39; and random.random() &gt;= 0.1:
     raise ValueError(&#39;chandan is not a valid name&#39;)
   time.sleep(1)
-  return f&#34;Hello {name}!&#34;</code></pre>
+  print(f&#34;Hello {name}!&#34;)
+  return</code></pre>
 </details>
 </dd>
 </dl>

docs/embgam/embed.html

@@ -86,8 +86,6 @@ <h1 class="title">Module <code>imodelsx.embgam.embed</code></h1>
     fit_with_ngram_decomposition: bool = True,
 ):
     &#34;&#34;&#34;Get summed embeddings for a single example
-    Note: this function gets called many times, so don&#39;t want to do things like load a model here
-
 
     Params
     ------
@@ -104,7 +102,7 @@ <h1 class="title">Module <code>imodelsx.embgam.embed</code></h1>
     nlp_chunks
         if parsing is not empty string, a parser that extracts specific ngrams
     fit_with_ngram_decomposition
-        whether to fit the model with ngram decomposition (if not just use the stsandard sentence)
+        whether to fit the model with ngram decomposition (if not just use the standard sentence)
     &#34;&#34;&#34;
     if dataset_key_text is not None:
         sentence = example[dataset_key_text]
@@ -170,8 +168,7 @@ <h2 class="section-title" id="header-functions">Functions</h2>
 <span>def <span class="ident">embed_and_sum_function</span></span>(<span>example, model, ngrams: int, tokenizer_embeddings, tokenizer_ngrams, checkpoint: str, dataset_key_text: str = None, layer: str = 'last_hidden_state', padding: bool = True, parsing: str = '', nlp_chunks=None, all_ngrams: bool = False, fit_with_ngram_decomposition: bool = True)</span>
 </code></dt>
 <dd>
-<div class="desc"><p>Get summed embeddings for a single example
-Note: this function gets called many times, so don't want to do things like load a model here</p>
+<div class="desc"><p>Get summed embeddings for a single example</p>
 <h2 id="params">Params</h2>
 <p>ngrams: int
 What order of ngrams to use (1 for unigrams, 2 for bigrams, &hellip;)
@@ -186,7 +183,7 @@ <h2 id="params">Params</h2>
 nlp_chunks
 if parsing is not empty string, a parser that extracts specific ngrams
 fit_with_ngram_decomposition
-whether to fit the model with ngram decomposition (if not just use the stsandard sentence)</p></div>
+whether to fit the model with ngram decomposition (if not just use the standard sentence)</p></div>
 <details class="source">
 <summary>
 <span>Expand source code</span>
@@ -207,8 +204,6 @@ <h2 id="params">Params</h2>
     fit_with_ngram_decomposition: bool = True,
 ):
     &#34;&#34;&#34;Get summed embeddings for a single example
-    Note: this function gets called many times, so don&#39;t want to do things like load a model here
-
 
     Params
     ------
@@ -225,7 +220,7 @@ <h2 id="params">Params</h2>
     nlp_chunks
         if parsing is not empty string, a parser that extracts specific ngrams
     fit_with_ngram_decomposition
-        whether to fit the model with ngram decomposition (if not just use the stsandard sentence)
+        whether to fit the model with ngram decomposition (if not just use the standard sentence)
     &#34;&#34;&#34;
     if dataset_key_text is not None:
         sentence = example[dataset_key_text]

docs/index.html

@@ -171,6 +171,10 @@ <h1 id="related-work">Related work</h1>
 <section>
 <h2 class="section-title" id="header-submodules">Sub-modules</h2>
 <dl>
+<dt><code class="name"><a title="imodelsx.cache_save_utils" href="cache_save_utils.html">imodelsx.cache_save_utils</a></code></dt>
+<dd>
+<div class="desc"></div>
+</dd>
 <dt><code class="name"><a title="imodelsx.d3" href="d3/index.html">imodelsx.d3</a></code></dt>
 <dd>
 <div class="desc"></div>
@@ -191,10 +195,18 @@ <h2 class="section-title" id="header-submodules">Sub-modules</h2>
 <dd>
 <div class="desc"></div>
 </dd>
+<dt><code class="name"><a title="imodelsx.linear_finetune" href="linear_finetune.html">imodelsx.linear_finetune</a></code></dt>
+<dd>
+<div class="desc"><p>Simple scikit-learn interface for Emb-GAM …</p></div>
+</dd>
 <dt><code class="name"><a title="imodelsx.metrics" href="metrics.html">imodelsx.metrics</a></code></dt>
 <dd>
 <div class="desc"></div>
 </dd>
+<dt><code class="name"><a title="imodelsx.process_results" href="process_results.html">imodelsx.process_results</a></code></dt>
+<dd>
+<div class="desc"></div>
+</dd>
 <dt><code class="name"><a title="imodelsx.submit_utils" href="submit_utils.html">imodelsx.submit_utils</a></code></dt>
 <dd>
 <div class="desc"></div>
@@ -232,12 +244,15 @@ <h1>Index</h1>
 <ul id="index">
 <li><h3><a href="#header-submodules">Sub-modules</a></h3>
 <ul>
+<li><code><a title="imodelsx.cache_save_utils" href="cache_save_utils.html">imodelsx.cache_save_utils</a></code></li>
 <li><code><a title="imodelsx.d3" href="d3/index.html">imodelsx.d3</a></code></li>
 <li><code><a title="imodelsx.data" href="data.html">imodelsx.data</a></code></li>
 <li><code><a title="imodelsx.dummy_script" href="dummy_script.html">imodelsx.dummy_script</a></code></li>
 <li><code><a title="imodelsx.embgam" href="embgam/index.html">imodelsx.embgam</a></code></li>
 <li><code><a title="imodelsx.iprompt" href="iprompt/index.html">imodelsx.iprompt</a></code></li>
+<li><code><a title="imodelsx.linear_finetune" href="linear_finetune.html">imodelsx.linear_finetune</a></code></li>
 <li><code><a title="imodelsx.metrics" href="metrics.html">imodelsx.metrics</a></code></li>
+<li><code><a title="imodelsx.process_results" href="process_results.html">imodelsx.process_results</a></code></li>
 <li><code><a title="imodelsx.submit_utils" href="submit_utils.html">imodelsx.submit_utils</a></code></li>
 <li><code><a title="imodelsx.util" href="util.html">imodelsx.util</a></code></li>
 <li><code><a title="imodelsx.viz" href="viz.html">imodelsx.viz</a></code></li>

docs/iprompt/api.html

@@ -116,9 +116,7 @@ <h1 class="title">Module <code>imodelsx.iprompt.api</code></h1>
         &#39;output&#39;: output_strs,
         &#39;text&#39;: text_strs,
     })
-    if n_shots == 1:
-        dset = datasets.Dataset.from_pandas(df)
-    else:
+    if n_shots &gt; 1:
         d2 = defaultdict(list)
         for i in range(max_n_datapoints):
             all_shots = df.sample(n=n_shots, replace=False)
@@ -133,10 +131,11 @@ <h1 class="title">Module <code>imodelsx.iprompt.api</code></h1>
             d2[&#39;output&#39;].append(last_output)
             #
         df = pd.DataFrame.from_dict(d2)
-        # shuffle rows
+    # shuffle rows
+    if max_n_datapoints &lt; len(df):
         df = df.sample(n=max_n_datapoints, replace=False)
-        dset = datasets.Dataset.from_pandas(df)
-    print(&#39;loading model...&#39;)
+    dset = datasets.Dataset.from_pandas(df)
+    print(f&#39;iPrompt got {len(dset)} datapoints, now loading model...&#39;)
 
     model = model.to(device)
     dataloader = DataLoader(
@@ -551,6 +550,7 @@ <h1 class="title">Module <code>imodelsx.iprompt.api</code></h1>
         epoch_save_interval=epoch_save_interval,
         verbose=verbose,
     )
+    model = model.cpu()
     return r[&#39;prefixes&#39;], r
 
     # r = eval_model(args=args, r=r, dset=Dataset.from_dict(dset_test[:128]), model=model, tokenizer=tokenizer)
@@ -1085,6 +1085,7 @@ <h2 id="returns">Returns</h2>
         epoch_save_interval=epoch_save_interval,
         verbose=verbose,
     )
+    model = model.cpu()
     return r[&#39;prefixes&#39;], r
 
     # r = eval_model(args=args, r=r, dset=Dataset.from_dict(dset_test[:128]), model=model, tokenizer=tokenizer)</code></pre>
@@ -1146,9 +1147,7 @@ <h2 id="params">Params</h2>
         &#39;output&#39;: output_strs,
         &#39;text&#39;: text_strs,
     })
-    if n_shots == 1:
-        dset = datasets.Dataset.from_pandas(df)
-    else:
+    if n_shots &gt; 1:
         d2 = defaultdict(list)
         for i in range(max_n_datapoints):
             all_shots = df.sample(n=n_shots, replace=False)
@@ -1163,10 +1162,11 @@ <h2 id="params">Params</h2>
             d2[&#39;output&#39;].append(last_output)
             #
         df = pd.DataFrame.from_dict(d2)
-        # shuffle rows
+    # shuffle rows
+    if max_n_datapoints &lt; len(df):
         df = df.sample(n=max_n_datapoints, replace=False)
-        dset = datasets.Dataset.from_pandas(df)
-    print(&#39;loading model...&#39;)
+    dset = datasets.Dataset.from_pandas(df)
+    print(f&#39;iPrompt got {len(dset)} datapoints, now loading model...&#39;)
 
     model = model.to(device)
     dataloader = DataLoader(

docs/iprompt/autoprompt.html

@@ -126,15 +126,15 @@ <h1 class="title">Module <code>imodelsx.iprompt.autoprompt</code></h1>
         # os.makedirs(save_dir, exist_ok=True)
         # pickle.dump(self._prefix_pool, open(os.path.join(save_dir, &#39;prefix_pool.p&#39;), &#39;wb&#39;))
 
-        if self._do_final_reranking:
-            all_prefixes = self._prefix_pool.topk_all(
-                k=self._num_prefixes_to_test, min_occurrences=1)
+        all_prefixes = self._prefix_pool.topk_all(
+            k=self._num_prefixes_to_test, min_occurrences=2)
 
-            if not len(all_prefixes):
-                # In the case where we get no prefixes here (i.e. prompt generation
-                # only ran for a single step) just take anything from prefix pool.
-                all_prefixes = list(self._prefix_pool.prefixes)
+        if not len(all_prefixes):
+            # In the case where we get no prefixes here (i.e. prompt generation
+            # only ran for a single step) just take anything from prefix pool.
+            all_prefixes = random.choices(self._prefix_pool.prefixes, k=self._num_prefixes_to_test)
 
+        if self._do_final_reranking:
             all_losses, all_accuracies = self._test_prefixes(
                 prefixes=all_prefixes,
                 eval_dataloader=eval_dataloader,
@@ -148,14 +148,14 @@ <h1 class="title">Module <code>imodelsx.iprompt.autoprompt</code></h1>
                                 False, True]).reset_index()
         else:
             all_prefixes = list(self._prefix_pool.prefixes)
-            all_losses = [-1] * len(all_prefixes)
-            all_accuracies = [-1] * len(all_prefixes)
+            all_losses = [self._prefix_pool._avg_loss.get(p, -1) for p in all_prefixes]
+            all_accuracies = [self._prefix_pool._avg_accuracy.get(p, -1) for p in all_prefixes]
 
             df = pd.DataFrame(
                 zip(*[all_prefixes, all_losses, all_accuracies]),
                 columns=[&#39;prefix&#39;, &#39;loss&#39;, &#39;accuracy&#39;]
             )
-        # df = df.sort_values(by=&#39;loss&#39;, ascending=True).reset_index()
+        df = df.sort_values(by=&#39;accuracy&#39;, ascending=False).reset_index()
 
         df[&#39;prefix_str&#39;] = df[&#39;prefix&#39;].map(self.tokenizer.decode)
         df[&#39;n_queries&#39;] = df[&#39;prefix&#39;].map(
@@ -430,15 +430,15 @@ <h2 class="section-title" id="header-classes">Classes</h2>
         # os.makedirs(save_dir, exist_ok=True)
         # pickle.dump(self._prefix_pool, open(os.path.join(save_dir, &#39;prefix_pool.p&#39;), &#39;wb&#39;))
 
-        if self._do_final_reranking:
-            all_prefixes = self._prefix_pool.topk_all(
-                k=self._num_prefixes_to_test, min_occurrences=1)
+        all_prefixes = self._prefix_pool.topk_all(
+            k=self._num_prefixes_to_test, min_occurrences=2)
 
-            if not len(all_prefixes):
-                # In the case where we get no prefixes here (i.e. prompt generation
-                # only ran for a single step) just take anything from prefix pool.
-                all_prefixes = list(self._prefix_pool.prefixes)
+        if not len(all_prefixes):
+            # In the case where we get no prefixes here (i.e. prompt generation
+            # only ran for a single step) just take anything from prefix pool.
+            all_prefixes = random.choices(self._prefix_pool.prefixes, k=self._num_prefixes_to_test)
 
+        if self._do_final_reranking:
             all_losses, all_accuracies = self._test_prefixes(
                 prefixes=all_prefixes,
                 eval_dataloader=eval_dataloader,
@@ -452,14 +452,14 @@ <h2 class="section-title" id="header-classes">Classes</h2>
                                 False, True]).reset_index()
         else:
             all_prefixes = list(self._prefix_pool.prefixes)
-            all_losses = [-1] * len(all_prefixes)
-            all_accuracies = [-1] * len(all_prefixes)
+            all_losses = [self._prefix_pool._avg_loss.get(p, -1) for p in all_prefixes]
+            all_accuracies = [self._prefix_pool._avg_accuracy.get(p, -1) for p in all_prefixes]
 
             df = pd.DataFrame(
                 zip(*[all_prefixes, all_losses, all_accuracies]),
                 columns=[&#39;prefix&#39;, &#39;loss&#39;, &#39;accuracy&#39;]
             )
-        # df = df.sort_values(by=&#39;loss&#39;, ascending=True).reset_index()
+        df = df.sort_values(by=&#39;accuracy&#39;, ascending=False).reset_index()
 
         df[&#39;prefix_str&#39;] = df[&#39;prefix&#39;].map(self.tokenizer.decode)
         df[&#39;n_queries&#39;] = df[&#39;prefix&#39;].map(
@@ -682,15 +682,15 @@ <h3>Methods</h3>
     # os.makedirs(save_dir, exist_ok=True)
     # pickle.dump(self._prefix_pool, open(os.path.join(save_dir, &#39;prefix_pool.p&#39;), &#39;wb&#39;))
 
-    if self._do_final_reranking:
-        all_prefixes = self._prefix_pool.topk_all(
-            k=self._num_prefixes_to_test, min_occurrences=1)
+    all_prefixes = self._prefix_pool.topk_all(
+        k=self._num_prefixes_to_test, min_occurrences=2)
 
-        if not len(all_prefixes):
-            # In the case where we get no prefixes here (i.e. prompt generation
-            # only ran for a single step) just take anything from prefix pool.
-            all_prefixes = list(self._prefix_pool.prefixes)
+    if not len(all_prefixes):
+        # In the case where we get no prefixes here (i.e. prompt generation
+        # only ran for a single step) just take anything from prefix pool.
+        all_prefixes = random.choices(self._prefix_pool.prefixes, k=self._num_prefixes_to_test)
 
+    if self._do_final_reranking:
         all_losses, all_accuracies = self._test_prefixes(
             prefixes=all_prefixes,
             eval_dataloader=eval_dataloader,
@@ -704,14 +704,14 @@ <h3>Methods</h3>
                             False, True]).reset_index()
     else:
         all_prefixes = list(self._prefix_pool.prefixes)
-        all_losses = [-1] * len(all_prefixes)
-        all_accuracies = [-1] * len(all_prefixes)
+        all_losses = [self._prefix_pool._avg_loss.get(p, -1) for p in all_prefixes]
+        all_accuracies = [self._prefix_pool._avg_accuracy.get(p, -1) for p in all_prefixes]
 
         df = pd.DataFrame(
             zip(*[all_prefixes, all_losses, all_accuracies]),
             columns=[&#39;prefix&#39;, &#39;loss&#39;, &#39;accuracy&#39;]
         )
-    # df = df.sort_values(by=&#39;loss&#39;, ascending=True).reset_index()
+    df = df.sort_values(by=&#39;accuracy&#39;, ascending=False).reset_index()
 
     df[&#39;prefix_str&#39;] = df[&#39;prefix&#39;].map(self.tokenizer.decode)
     df[&#39;n_queries&#39;] = df[&#39;prefix&#39;].map(

docs/iprompt/ipromptx.html

@@ -116,6 +116,7 @@ <h1 class="title">Module <code>imodelsx.iprompt.ipromptx</code></h1>
             tokenizer=self.tokenizer,
             criterion=pop_criterion, # &#39;loss&#39;  # in [&#39;loss&#39;, &#39;acc&#39;, &#39;combined&#39;]
             topk_strategy=pop_topk_strategy,
+            verbose=verbose,
         )
         # Suff to track for early stopping
         self._early_stopping_steps = early_stopping_steps
@@ -254,8 +255,8 @@ <h1 class="title">Module <code>imodelsx.iprompt.ipromptx</code></h1>
 
     def _get_population_and_random_generations(self, full_text_ids: torch.Tensor) -&gt; torch.Tensor:
         population_pool = self._select_pop_topk(k=self._topk_pop_sample)
-        if self._iprompt_verbose:
-            print(&#34;population_pool:&#34;, [self.tokenizer.decode(p) for p in population_pool])
+        # if self._iprompt_verbose:
+            # print(&#34;population_pool:&#34;, [self.tokenizer.decode(p) for p in population_pool])
         population = random.sample(population_pool, self._pop_size)
         population = torch.tensor(population).to(device)
 
@@ -569,6 +570,7 @@ <h2 class="section-title" id="header-classes">Classes</h2>
             tokenizer=self.tokenizer,
             criterion=pop_criterion, # &#39;loss&#39;  # in [&#39;loss&#39;, &#39;acc&#39;, &#39;combined&#39;]
             topk_strategy=pop_topk_strategy,
+            verbose=verbose,
         )
         # Suff to track for early stopping
         self._early_stopping_steps = early_stopping_steps
@@ -707,8 +709,8 @@ <h2 class="section-title" id="header-classes">Classes</h2>
 
     def _get_population_and_random_generations(self, full_text_ids: torch.Tensor) -&gt; torch.Tensor:
         population_pool = self._select_pop_topk(k=self._topk_pop_sample)
-        if self._iprompt_verbose:
-            print(&#34;population_pool:&#34;, [self.tokenizer.decode(p) for p in population_pool])
+        # if self._iprompt_verbose:
+            # print(&#34;population_pool:&#34;, [self.tokenizer.decode(p) for p in population_pool])
         population = random.sample(population_pool, self._pop_size)
         population = torch.tensor(population).to(device)