Merge branch 'master' of https://github.com/intsystems/predicator-function-for-neural-networks

Author: Vladimir Kurmanov

Udeneev2025Surrogate.pdf

@@ -11,7 +11,7 @@
     GraphNorm,
 )
 from torch_geometric.utils import dense_to_sparse
-from torch.optim.lr_scheduler import StepLR
+from torch.optim.lr_scheduler import StepLR, CosineAnnealingLR
 from tqdm.notebook import tqdm
 import matplotlib.pyplot as plt
 import numpy as np
@@ -347,10 +347,11 @@ def train_model_diversity(
     num_epochs,
     device="cpu",
     developer_mode=False,
+    final_lr=0.001
 ):
     model.to(device)
     train_losses, valid_losses = [], []
-    scheduler = StepLR(optimizer, step_size=5, gamma=0.5)
+    scheduler = CosineAnnealingLR(optimizer, num_epochs, eta_min=final_lr)
 
     for epoch in tqdm(range(num_epochs), desc="Training Progress"):
         # --------------------
@@ -424,7 +425,8 @@ def train_model_diversity(
         except ImportError:
             pass
 
-        plt.figure(figsize=(10, 5))
+        plt.figure(figsize=(12, 6))
+        plt.rc('font', size=20)
         plt.plot(range(1, len(train_losses)+1), train_losses, marker='o', label='Train Loss')
         plt.plot(range(1, len(valid_losses)+1), valid_losses, marker='s', label='Valid Loss')
         plt.xlabel('Epoch')
@@ -449,12 +451,13 @@ def train_model_accuracy(
     num_epochs,
     device="cpu",
     developer_mode=False,
+    final_lr=0.001,
 ):
     model.to(device)
     train_losses = []
     valid_losses = []
 
-    scheduler = StepLR(optimizer, step_size=5, gamma=0.5)
+    scheduler = CosineAnnealingLR(optimizer, num_epochs, eta_min=final_lr)
 
     for epoch in tqdm(range(num_epochs), desc="Training Progress"):
         model.train()
@@ -511,6 +514,7 @@ def train_model_accuracy(
             pass
 
         plt.figure(figsize=(12, 6))
+        plt.rc('font', size=20)
         tmp_train_losses = np.array(train_losses)
         tmp_valid_losses = np.array(valid_losses)
         plt.plot(range(1, len(tmp_train_losses) + 1), tmp_train_losses * 1e4, marker="o", label="Train Loss")

code/best_models.zip

@@ -1,6 +1,6 @@
-<mxfile host="app.diagrams.net" agent="Mozilla/5.0 (X11; Linux x86_64; rv:138.0) Gecko/20100101 Firefox/138.0" version="27.0.2">
+<mxfile host="app.diagrams.net" agent="Mozilla/5.0 (X11; Linux x86_64; rv:138.0) Gecko/20100101 Firefox/138.0" version="27.0.5">
   <diagram name="GAT Architecture" id="gat-diagram">
-    <mxGraphModel dx="1072" dy="577" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+    <mxGraphModel dx="1554" dy="845" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
       <root>
         <mxCell id="0" />
         <mxCell id="1" parent="0" />
@@ -17,7 +17,7 @@
           <mxGeometry x="130" y="480" width="160" height="40" as="geometry" />
         </mxCell>
         <mxCell id="fc1" value="&lt;div&gt;Linear&lt;/div&gt;" style="shape=rectangle;rounded=1;whiteSpace=wrap;html=1;fillColor=#F5F5F5;" parent="1" vertex="1">
-          <mxGeometry x="400" y="300" width="160" height="30" as="geometry" />
+          <mxGeometry x="400" y="300" width="160" height="40" as="geometry" />
         </mxCell>
         <mxCell id="fc_norm" value="&lt;div&gt;Leaky ReLU&lt;/div&gt;&lt;div&gt;LayerNorm&lt;/div&gt;" style="shape=rectangle;rounded=1;whiteSpace=wrap;html=1;fillColor=#D5E8D4;" parent="1" vertex="1">
           <mxGeometry x="400" y="360" width="160" height="40" as="geometry" />
@@ -48,19 +48,19 @@
         <mxCell id="e13" style="edgeStyle=orthogonalEdgeStyle;rounded=0;" parent="1" source="fc2" target="out" edge="1">
           <mxGeometry relative="1" as="geometry" />
         </mxCell>
-        <mxCell id="1mhNudxIzqTfyHtObyKU-1" value="&lt;div&gt;&lt;font style=&quot;font-size: 17px;&quot;&gt;N×&lt;/font&gt;&lt;/div&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
+        <mxCell id="1mhNudxIzqTfyHtObyKU-1" value="&lt;div&gt;&lt;font style=&quot;font-size: 17px;&quot;&gt;N×&lt;/font&gt;&lt;/div&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
           <mxGeometry x="50" y="360" width="60" height="30" as="geometry" />
         </mxCell>
-        <mxCell id="1mhNudxIzqTfyHtObyKU-2" value="&lt;div&gt;Output Layer&lt;/div&gt;" style="shape=rectangle;rounded=1;whiteSpace=wrap;html=1;fillColor=#DAE8FC;" vertex="1" parent="1">
+        <mxCell id="1mhNudxIzqTfyHtObyKU-2" value="&lt;div&gt;Output Layer&lt;/div&gt;" style="shape=rectangle;rounded=1;whiteSpace=wrap;html=1;fillColor=#DAE8FC;" parent="1" vertex="1">
           <mxGeometry x="400" y="550" width="160" height="40" as="geometry" />
         </mxCell>
-        <mxCell id="1mhNudxIzqTfyHtObyKU-3" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="out" target="1mhNudxIzqTfyHtObyKU-2">
+        <mxCell id="1mhNudxIzqTfyHtObyKU-3" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" parent="1" source="out" target="1mhNudxIzqTfyHtObyKU-2" edge="1">
           <mxGeometry width="50" height="50" relative="1" as="geometry">
             <mxPoint x="340" y="470" as="sourcePoint" />
             <mxPoint x="390" y="420" as="targetPoint" />
           </mxGeometry>
         </mxCell>
-        <mxCell id="1mhNudxIzqTfyHtObyKU-4" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="pool" target="fc1">
+        <mxCell id="1mhNudxIzqTfyHtObyKU-4" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" parent="1" source="pool" target="fc1" edge="1">
           <mxGeometry width="50" height="50" relative="1" as="geometry">
             <mxPoint x="340" y="510" as="sourcePoint" />
             <mxPoint x="390" y="460" as="targetPoint" />
@@ -72,7 +72,7 @@
             </Array>
           </mxGeometry>
         </mxCell>
-        <mxCell id="1mhNudxIzqTfyHtObyKU-8" value="" style="shape=curlyBracket;whiteSpace=wrap;html=1;rounded=1;labelPosition=left;verticalLabelPosition=middle;align=right;verticalAlign=middle;" vertex="1" parent="1">
+        <mxCell id="1mhNudxIzqTfyHtObyKU-8" value="" style="shape=curlyBracket;whiteSpace=wrap;html=1;rounded=1;labelPosition=left;verticalLabelPosition=middle;align=right;verticalAlign=middle;" parent="1" vertex="1">
           <mxGeometry x="100" y="290" width="20" height="180" as="geometry" />
         </mxCell>
       </root>