consoleGA Blueprint Release (#574)

* Update blueprints for launch (#553)

* add evaluate step

* update blueprints for GA

* new blueprint cards for DD & SS

- Pulled in notebooks and new structure for DD notebooks from main
- Added new blueprint cards
-- DD multi-turn, rag-eval, text-to-code
-- SS hipaa and gdpr
- Added details pages
- Added sample datasets for SS

* fixed details file link and added placeholder details for transfor

* updates to links, new transform details

* update copy

* minor changes

* tweaks

* still tweaking copy

* copy changes

* Navigator removal & Transform config updates (#558)

* Add otnotes5 data sample and preview (#562)

* Lots of blueprint updates for GA & v2 (#561)

* Many blueprint updates for GA

* Adding rope_scaling_factor to the default config

* Fixing where rope_scaling_factor goes

* Updating .md files to remove outdated info & v1 notebook links

* Reducing tabFT num records to 1000 default for faster runs

* Removing old images & icons

* Fixing name of file

* Trying to get rid of errors

* underscore to hyphen typo

* Changing model blueprints to use a workflow config (#563)

* Changing model blueprints to use a workflow config

* Updating to use new evaluate task name

* Updating task names (#564)

* updating task names

* Swapping order for consistency

* Fixing hyphen vs underscore typo

* Updated dd image for blueprint card (#569)

* Updated dd image

* Updated blueprint card to add 101 notebook

* Updated gretel.json

---------

Co-authored-by: Kirit93 <kthadaka@nvidia.com>

* Updated notebook links (#572)

Co-authored-by: Kirit93 <kthadaka@nvidia.com>

* Fixing small typo (#573)

* Updating num_records to be 1000 across the board (#575)

* Update gretel.json with DD notebook link (#580)

* Update gretel.json (#583)

* nit fixes to workflow blueprints (#585)

* nit: make spacing consistent

* fix num_records for one blueprint

* change hyphen to underscore in task names

* add collab links pointing to main and 101 notebook (#587)

* Update workflow tests

* Improve tests

* Ensure we catch errors that are returned with 200

* Also log in "200 with error" responses

---------

Co-authored-by: Yamini <yamini@users.noreply.github.com>
Co-authored-by: alexahaushalter <alexahaushalter@hotmail.com>
Co-authored-by: Kirit Thadaka <kirit.thadaka@gmail.com>
Co-authored-by: Kirit93 <kthadaka@nvidia.com>
Co-authored-by: Nina Xu <nina.ning.xu@gmail.com>
Co-authored-by: Matt Kornfield <mckornfield@gmail.com>
Co-authored-by: Henrique Tolentino <htolentino@nvidia.com>

Author: 7 people

config_templates/gretel/synthetics/navigator-ft-differential-privacy.yml

@@ -17,7 +17,7 @@ models:
     order_training_examples_by: null
 
     generate:
-        num_records: 5000
+        num_records: 1000
 
         # With DP, enabling structured generation can help with
         # increasing the percentage of valid records.

config_templates/gretel/synthetics/navigator-ft.yml

@@ -17,7 +17,7 @@ models:
       order_training_examples_by: null
 
       generate:
-        num_records: 5000
+        num_records: 1000
 
       params:
         # The parameter below is a proxy for training time.
@@ -28,4 +28,13 @@ models:
         # (we downsample), larger (we resample), or the same
         # size as your input dataset. A starting value to
         # experiment with is 25,000.
-        num_input_records_to_sample: auto
+        num_input_records_to_sample: auto
+
+        # Scale the base LLM's context length by this factor
+        # using RoPE scaling to handle datasets with more
+        # columns, or datasets containing groups with more
+        # than a few records. You can try increasing the
+        # rope_scaling_factor (you could first try the value 2)
+        # if you hit an error for maximum tokens. It must be
+        # an integer value. The default is 1 and maximum is 6. 
+        rope_scaling_factor: 1

config_templates/gretel/tasks/tabular_ft__default.yaml

@@ -3,31 +3,38 @@ name: default
 task:
   name: tabular_ft
   config:
-      train:
-        # Optionally group records by the column(s) set below.
-        # This is useful if you need to maintain correlations
-        # across multiple records. Otherwise, the training
-        # assumes records are independent.
-        group_training_examples_by: null
+    train:
+      # Optionally group records by the column(s) set below.
+      # This is useful if you need to maintain correlations
+      # across multiple records. Otherwise, the training
+      # assumes records are independent.
+      group_training_examples_by: null
 
-        # Optionally order records by the column set below.
-        # This is useful if your records are sequential.
-        # Note that this parameter can only be used when
-        # your records are grouped using the above parameter.
-        order_training_examples_by: null
+      # Optionally order records by the column set below.
+      # This is useful if your records are sequential.
+      # Note that this parameter can only be used when
+      # your records are grouped using the above parameter.
+      order_training_examples_by: null
 
-        params:
-          # The parameter below is a proxy for training time.
-          # If set to 'auto', we will automatically choose an
-          # appropriate value. An integer value will set the
-          # number of records from the input dataset that the
-          # model will see during training. It can be smaller
-          # (we downsample), larger (we resample), or the same
-          # size as your input dataset. A starting value to
-          # experiment with is 25,000.
-          num_input_records_to_sample: auto
-
-      generate:
-        num_records: 5000
+      params:
+        # The parameter below is a proxy for training time.
+        # If set to 'auto', we will automatically choose an
+        # appropriate value. An integer value will set the
+        # number of records from the input dataset that the
+        # model will see during training. It can be smaller
+        # (we downsample), larger (we resample), or the same
+        # size as your input dataset. A starting value to
+        # experiment with is 25,000.
+        num_input_records_to_sample: auto
 
+        # Scale the base LLM's context length by this factor
+        # using RoPE scaling to handle datasets with more
+        # columns, or datasets containing groups with more
+        # than a few records. You can try increasing the
+        # rope_scaling_factor (you could first try the value 2)
+        # if you hit an error for maximum tokens. It must be
+        # an integer value. The default is 1 and maximum is 6. 
+        rope_scaling_factor: 1
 
+    generate:
+      num_records: 1000

config_templates/gretel/tasks/tabular_ft__differential_privacy.yaml

@@ -3,46 +3,53 @@ name: differential_privacy
 task:
   name: tabular_ft
   config:
-      train:
-        # Optionally group records by the column(s) set below.
-        # This is useful if you need to maintain correlations
-        # across multiple records. Otherwise, the training
-        # assumes records are independent.
-        group_training_examples_by: null
-
-        # Optionally order records by the column set below.
-        # This is useful if your records are sequential.
-        # Note that this parameter can only be used when
-        # your records are grouped using the above parameter.
-        order_training_examples_by: null
-
-        privacy_params:
-            dp: true
-
-            # Defines the privacy budget - the larger the value, the
-            # less privacy we get. A value between 2 and 8 is deemed
-            # reasonable, usually.
-            epsilon: 8
-
-        params:
-          # The parameter below is a proxy for training time.
-          # If set to 'auto', we will automatically choose an
-          # appropriate value. An integer value will set the
-          # number of records from the input dataset that the
-          # model will see during training. It can be smaller
-          # (we downsample), larger (we resample), or the same
-          # size as your input dataset. A starting value to
-          # experiment with is 25,000.
-          num_input_records_to_sample: auto
-
-          # You can try increasing this until you run out-of-memory.
-          batch_size: 4
-
-      generate:
-        num_records: 5000
-
-        # With DP, enabling structured generation can help with
-        # increasing the percentage of valid records.
-        use_structured_generation: true
-
-
+    train:
+      # Optionally group records by the column(s) set below.
+      # This is useful if you need to maintain correlations
+      # across multiple records. Otherwise, the training
+      # assumes records are independent.
+      group_training_examples_by: null
+
+      # Optionally order records by the column set below.
+      # This is useful if your records are sequential.
+      # Note that this parameter can only be used when
+      # your records are grouped using the above parameter.
+      order_training_examples_by: null
+
+      privacy_params:
+        dp: true
+
+        # Defines the privacy budget - the larger the value, the
+        # less privacy we get. A value between 2 and 8 is deemed
+        # reasonable, usually.
+        epsilon: 8
+
+      params:
+        # The parameter below is a proxy for training time.
+        # If set to 'auto', we will automatically choose an
+        # appropriate value. An integer value will set the
+        # number of records from the input dataset that the
+        # model will see during training. It can be smaller
+        # (we downsample), larger (we resample), or the same
+        # size as your input dataset. A starting value to
+        # experiment with is 25,000.
+        num_input_records_to_sample: auto
+
+        # Scale the base LLM's context length by this factor
+        # using RoPE scaling to handle datasets with more
+        # columns, or datasets containing groups with more
+        # than a few records. You can try increasing the
+        # rope_scaling_factor (you could first try the value 2)
+        # if you hit an error for maximum tokens. It must be
+        # an integer value. The default is 1 and maximum is 6. 
+        rope_scaling_factor: 1
+
+        # You can try increasing this until you run out-of-memory.
+        batch_size: 4
+
+    generate:
+      num_records: 1000
+
+      # With DP, enabling structured generation can help with
+      # increasing the percentage of valid records.
+      use_structured_generation: true

config_templates/gretel/tasks/tabular_gan__default.yaml

@@ -13,4 +13,4 @@ task:
         batch_size: auto
         auto_transform_datetimes: False
     generate:
-      num_records: 5000
+      num_records: 1000

config_templates/gretel/tasks/text_ft__default.yaml

@@ -13,5 +13,5 @@ task:
         lr_scheduler: "linear"
         learning_rate: 0.0001
     generate:
-      num_records: 80
+      num_records: 1000
       maximum_text_length: 100

config_templates/gretel/tasks/text_ft__differential_privacy.yaml

@@ -1,5 +1,5 @@
 schema_version: "1.0"
-name: default
+name: differential_privacy
 task:
   name: text_ft
   config:
@@ -23,5 +23,5 @@ task:
         epsilon: 5  # Privacy budget (lower values = stronger privacy)
         delta: auto  # Probability of privacy leakage (auto-calculated)
     generate:
-      num_records: 80  # Number of records to generate
+      num_records: 1000  # Number of records to generate
       maximum_text_length: 128  # Maximum length of generated texts in tokens