Merge pull request #11 from sbsos/feature/Replotting

Adding support for replotting

Author: sbsos

hotplotterclient/controller.py

@@ -5,6 +5,7 @@
 import requests
 import json
 import time
+from features import features
 
 
 class plotProcessDto(object):
@@ -26,13 +27,15 @@ def __init__(self, processId, phase, step, plotNumber, destinationDrive, plottin
 
 
 class payload(object):
-
+    
     def __init__(self):
         self.plottingStatuses = []
         self.hardDrives = []
         self.username = ""
         self.plotterKey = ""
-    
+        self.features = []
+        self.updateHardDrives = False
+        
     def addHardDrives(self, hardDrives):
         self.hardDrives = hardDrives
 
@@ -47,6 +50,9 @@ def addAuth(self, username, plotterKey):
         self.username = username
         self.plotterKey = plotterKey
 
+    def addFeatures(self):
+        self.features = features().getFeatures()
+    
     def toJSON(self):
         return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=4)
 
@@ -55,10 +61,11 @@ class master_controller(object):
     plot_controllers = []
     plot_controller = plot_process_controller()
     hd_controller = hard_drive_controller()
+    lastHardDriveHash = ''
     username = ''
     plotterKey = ''
     api_url = 'https://api.hotplotter.com/api/Plotter/UpdateStatus'
-
+    
     def __init__(self, username, plotterKey):
         self.username = username
         self.plotterKey = plotterKey
@@ -86,16 +93,25 @@ def transformToPlotConfig(self, queuedPlot):
         config['plotter_type'] = queuedPlot['plotterType']
         config['pool_type'] = queuedPlot['poolType']
         config['client_identifier'] = queuedPlot['clientIdentifier']
+        config['replot'] = queuedPlot['replot']
 
         return config
 
     def heart_beat(self):
         heartbeat_interval = 120.0
 
         postData = payload()
-        postData.addHardDrives(self.hd_controller.get_hard_drives())
+        hardDriveStatuses = self.hd_controller.get_hard_drives()
+        updateHardDrives = hash(tuple(hardDriveStatuses)) != self.lastHardDriveHash
+        if(updateHardDrives):
+            postData.addHardDrives(hardDriveStatuses)
+            self.lastHardDriveHash = hash(tuple(hardDriveStatuses))
+            postData.updateHardDrives = True
+            
         postData.addPlots(self.plot_controller)
         postData.addAuth(self.username, self.plotterKey)
+        postData.addFeatures()
+
         try:
             x = requests.post(self.api_url, data=postData.toJSON(), headers={"Content-Type": "application/json"})
             if(x.ok):

hotplotterclient/features.py

@@ -0,0 +1,5 @@
+class features(object):
+    
+    def getFeatures(self):
+        return ['replot']
+

hotplotterclient/plotter.py

@@ -3,6 +3,7 @@
 import datetime
 import os
 import shlex
+from replotting import replotting
 
 class plot_process_monitor(object):
     proc = None
@@ -16,19 +17,19 @@ class plot_process_monitor(object):
     file_name = ""
     step = ""
     plot_number = 0
-    number_of_iterations = 0
     client_identifier = ""
+    replotting = None
 
-    def __init__(self, proc, temp_drive, temp_drive_two, destination_drive, number_of_iterations, client_identifier):
+    def __init__(self, proc, temp_drive, temp_drive_two, destination_drive, client_identifier, replotting):
         self.proc = proc
         self.temp_drive = temp_drive
         self.temp_drive_two = temp_drive_two
         self.destination_drive = destination_drive
         self.file_name = str(self.proc.pid) + ".txt"
         self.start_time = datetime.datetime.now()
-        self.number_of_iterations = number_of_iterations
         self.client_identifier = client_identifier
-
+        self.replotting = replotting
+        
     @property
     def plotting_drives(self):
         joined_drive_names = self.temp_drive
@@ -47,6 +48,13 @@ def start_monitoring(self):
 
             if "Starting phase 1/4".lower() in line.lower() or "plot name" in line.lower():
                 self.plot_number += 1
+                #client_identifier has significance. If a user has multiple plot processes running to same destination drive, not all want might want to replot OG. client_identifier is correlated to the individual plot kickoff, which contains if replotting is enabled or not
+                deletedPlot = self.replotting.deletePlot(self.destination_drive, self.client_identifier)
+                replottingEnabled = deletedPlot == None
+                if replottingEnabled:
+                    if deletedPlot == False:
+                        #might want to check HD space at this point. If none, we kick out. If there is still enough for the size of the plot let it continue.
+                        placeHolder = "placeHolder"
 
             parsed_phase = self.get_phase(line)
             if parsed_phase != 0:
@@ -88,7 +96,8 @@ class settings_config(object):
     plotter_type = ""
     pool_type = "OriginalPlot"
     client_identifier = ""
-
+    replot = False
+    
     def __init__(self, dictionary_config):
         self.drives = dictionary_config["drives"]
         self.stagger_type = dictionary_config['stagger_type']
@@ -105,13 +114,15 @@ def __init__(self, dictionary_config):
         self.buckets = dictionary_config['buckets']
         self.pool_type = dictionary_config['pool_type']    
         self.plotter_type = dictionary_config['plotter_type']
+        self.replot = dictionary_config['replot']
 
 
 class plot_process_controller(object):
     list_of_plotting_processes = []
     log_file_name = "log_file_"
     active_plot_monitors = []
-
+    replotting = replotting()
+    
     def add_monitor(self, monitor):
         self.active_plot_monitors.add(monitor)
 
@@ -163,7 +174,9 @@ def start_plot(self, config, plot_config, iteration):
 
         plot_config["parallel_plots"] -= 1
 
-        monitor = plot_process_monitor(proc, plot_config["temp_drive"], plot_config["temp_drive_two"], plot_config["destination_drive"], iteration, plot_config["client_identifier"])
+        #we check if enabled inside
+        self.replotting.addDrivePlots(plot_config["destination_drive"], plot_config["client_identifier"], config.replot)
+        monitor = plot_process_monitor(proc, plot_config["temp_drive"], plot_config["temp_drive_two"], plot_config["destination_drive"], plot_config["client_identifier"], self.replotting)
 
         thread = threading.Thread(target = monitor.start_monitoring)
         thread.daemon = True
@@ -219,12 +232,18 @@ def begin_mad_max_plot(self):
         command = self.build_mad_max_command_safe()
         print('#Beginning Temp Plot: ' + self.plotting_drives + ' Plotting to ' + self.final_drive + " using Chia Plotter")
         print("Count: " + str(self.config.count) + " Threads: " + str(self.config.threads) + " Buckets: " + str(self.config.buckets) + " TempDrive: " + str(self.plotting_drives) + " Final Drive: " + str(self.final_drive) + " Farm Key: " + self.config.farm_key + " Pool Key: " + self.config.pool_key)
-        proc = subprocess.Popen(command, stdout=subprocess.PIPE, text=True)
+        try:
+            proc = subprocess.Popen(command, stdout=subprocess.PIPE, text=True)
+        except OSError as e:
+            raise Exception('Please make sure you have permission to plot in the assigned directory, and that you have chia_plot in your PATH')
         return proc
 
     def begin_chia_plot(self):
         command = self.build_chia_plot_command_safe()
         print('#Beginning Temp Plot: ' + self.temp_drive + ' Plotting to ' + self.final_drive + " using Chia Plotter")
         print("Count: " + str(self.config.count) + " Ram: " + str(self.config.ram) + " Threads: " + str(self.config.threads) + " Size: " + str(self.config.size) + " TempDrive: " + str(self.temp_drive) + " Final Drive: " + str(self.final_drive) + " Farm Key: " + self.config.farm_key + " Pool Key: " + self.config.pool_key)
-        proc = subprocess.Popen(command, stdout=subprocess.PIPE, text=True)
+        try:
+            proc = subprocess.Popen(command, stdout=subprocess.PIPE, text=True)
+        except OSError as e:
+            raise Exception('Please make sure you have permission to plot in the assigned directory, and that you have chia in your PATH')
         return proc

hotplotterclient/replotting.py

@@ -0,0 +1,93 @@
+import os
+import threading
+
+class drivePlots(object):
+    driveName = ""
+    plots = []
+    plotTypes = ""
+    offset = 58
+    def __init__(self, driveName, plotTypes):
+        self.driveName = driveName
+        self.populatePlots(PlotTypes.OgPlots)
+
+    def filterPlotsInDirectory(self, files):
+        plots = filter(lambda c: 'plot' in c, files)
+        return plots
+
+    def populatePlots(self, filterPlotType):
+        files = os.listdir(self.driveName)
+        plots = self.filterPlotsInDirectory(files)
+        for plot in plots:
+            plotWithFullPath = ''
+            if self.driveName[-1] != '/':
+                plotWithFullPath = self.driveName + '/' + plot
+            else:
+                plotWithFullPath = self.driveName + plot
+
+            with open(plotWithFullPath, "rb") as plotFile:
+                plotFile.read(self.offset)
+                res = plotFile.read(2)
+
+                #Two Byte to int, w/ big to achieve their Utility.TwoByteToInt
+                length = res[1] >> res[0]
+                
+                plotType = self.getPlotType(length)
+                if plotType == filterPlotType:
+                    self.plots.append(plotWithFullPath)
+                    
+    def getPlotType(self, memoLength):
+        if memoLength == 112:
+            return PlotTypes.NftPlots
+        
+        elif memoLength == 128:
+            return PlotTypes.OgPlots
+        
+        #I did not understand this as well as I thought
+        return None
+    
+    def getPlot(self):
+        plotName = self.plots.pop()
+        return plotName
+                
+    
+class replotting(object):
+    drivePlotsLookup = {}
+    shouldReplot = False
+    lock = threading.Lock()
+    
+    def __init__(self):
+        return None
+        
+    #Only call upon new plot starting
+    def addDrivePlots(self, drivePath, clientIdentifier, enabled):        
+        if enabled:
+            dp = drivePlots(drivePath, PlotTypes.OgPlots)
+            self.drivePlotsLookup[drivePath + clientIdentifier] = dp
+        
+
+    def getPlotToDelete(self, drive, clientIdentifier):
+        with self.lock:
+            plotLookUp = self.drivePlotsLookup[drive + clientIdentifier]
+            plotName = plotLookUp.getPlot()
+            return plotName
+    
+    def deletePlot(self, drive, clientIdentifier):
+        if drive + clientIdentifier in self.drivePlotsLookup:
+            plotWithFullPath = self.getPlotToDelete(drive, clientIdentifier)
+            if plotWithFullPath is not None:
+                os.remove(plotWithFullPath)
+                return True
+            else:
+                return False
+        return None
+    
+class PlotTypes(object):
+    OgPlots = "OgPlots"
+    NftPlots = "NftPlots"
+
+    
+if __name__ == '__main__':
+    replotting = replotting()
+    replotting.addDrivePlots('A:/Plots', '1', True)
+    plotName = replotting.deletePlot('A:/Plots', '1')
+    print(plotName)