Merge pull request #1319 from PLC-lang/embecosm-online-chane

Initial online change support
- Adds an online change flag
- Generates a GOT and moves references to an indirection when online change is active.

Author: ghaith

Cargo.lock

@@ -56,7 +56,6 @@ tempfile = "3"
 encoding_rs.workspace = true
 encoding_rs_io.workspace = true
 
-
 [lib]
 name = "rusty"
 path = "src/lib.rs"

Cargo.toml

@@ -17,6 +17,7 @@ plc_index = { path = "../plc_index" }
 
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
+toml = "0.5"
 clap = { version = "3.0", features = ["derive"] }
 rayon = "1.6.1"
 tempfile = "3"

compiler/plc_driver/Cargo.toml

@@ -5,7 +5,8 @@ use encoding_rs::Encoding;
 use plc_diagnostics::diagnostics::{diagnostics_registry::DiagnosticsConfiguration, Diagnostic};
 use std::{env, ffi::OsStr, num::ParseIntError, path::PathBuf};
 
-use plc::{output::FormatOption, ConfigFormat, DebugLevel, ErrorFormat, Target, Threads};
+use plc::output::FormatOption;
+use plc::{ConfigFormat, DebugLevel, ErrorFormat, Target, Threads, DEFAULT_GOT_LAYOUT_FILE};
 
 pub type ParameterError = clap::Error;
 
@@ -109,6 +110,22 @@ pub struct CompileParameters {
     ) ]
     pub hardware_config: Option<String>,
 
+    #[clap(
+        name = "got-layout-file",
+        long,
+        global = true,
+        help = "Obtain information about the current custom GOT layout from the given file if it exists.
+    Save information about the generated custom GOT layout to the given file.
+    Format is detected by extension.
+    Supported formats : json, toml",
+        default_value = DEFAULT_GOT_LAYOUT_FILE,
+        parse(try_from_str = validate_config),
+        // FIXME: For some reason, this does not work at the moment but it really should
+        // The binary behaves as expected but the tests fail
+        // requires = "online-change"
+    ) ]
+    pub got_layout_file: String,
+
     #[clap(
         name = "optimization",
         long,
@@ -202,6 +219,12 @@ pub struct CompileParameters {
     #[clap(name = "check", long, help = "Check only, do not generate any output", global = true)]
     pub check_only: bool,
 
+    #[clap(
+        long,
+        help = "Emit a binary with specific compilation information, suitable for online changes when ran under a conforming runtime"
+    )]
+    pub online_change: bool,
+
     #[clap(subcommand)]
     pub commands: Option<SubCommands>,
 }
@@ -388,6 +411,11 @@ impl CompileParameters {
         self.hardware_config.as_deref().and_then(get_config_format)
     }
 
+    pub fn got_layout_format(&self) -> ConfigFormat {
+        // It is safe to unwrap here, since the provided argument to `--got-online-change` has been checked with `validate_config`
+        get_config_format(&self.got_layout_file).unwrap()
+    }
+
     /// Returns the location where the build artifacts should be stored / output
     pub fn get_build_location(&self) -> Option<PathBuf> {
         match &self.commands {

compiler/plc_driver/src/cli.rs

@@ -19,7 +19,7 @@ use std::{
 
 use cli::{CompileParameters, ParameterError, SubCommands};
 use plc::{
-    codegen::CodegenContext, linker::LinkerType, output::FormatOption, DebugLevel, ErrorFormat,
+    codegen::CodegenContext, linker::LinkerType, output::FormatOption, DebugLevel, ErrorFormat, OnlineChange,
     OptimizationLevel, Target, Threads,
 };
 
@@ -54,6 +54,7 @@ pub struct CompileOptions {
     pub error_format: ErrorFormat,
     pub debug_level: DebugLevel,
     pub single_module: bool,
+    pub online_change: OnlineChange,
 }
 
 impl Default for CompileOptions {
@@ -67,6 +68,7 @@ impl Default for CompileOptions {
             error_format: ErrorFormat::None,
             debug_level: DebugLevel::None,
             single_module: false,
+            online_change: OnlineChange::Disabled,
         }
     }
 }
@@ -176,6 +178,14 @@ pub fn get_compilation_context<T: AsRef<str> + AsRef<OsStr> + Debug>(
         error_format: compile_parameters.error_format,
         debug_level: compile_parameters.debug_level(),
         single_module: compile_parameters.single_module,
+        online_change: if compile_parameters.online_change {
+            OnlineChange::Enabled {
+                file_name: compile_parameters.got_layout_file.clone(),
+                format: compile_parameters.got_layout_format(),
+            }
+        } else {
+            OnlineChange::Disabled
+        },
     };
 
     let libraries =

compiler/plc_driver/src/lib.rs

@@ -1,8 +1,10 @@
 use std::{
+    collections::HashMap,
     env,
     fs::{self, File},
     io::Write,
     path::{Path, PathBuf},
+    sync::Mutex,
 };
 
 use crate::{CompileOptions, LinkOptions};
@@ -22,7 +24,7 @@ use plc::{
         TypeAnnotator,
     },
     validation::Validator,
-    ConfigFormat, Target,
+    ConfigFormat, OnlineChange, Target,
 };
 use plc_diagnostics::{
     diagnostician::Diagnostician,
@@ -36,6 +38,42 @@ use project::{
 use rayon::prelude::*;
 use source_code::{source_location::SourceLocation, SourceContainer};
 
+use serde_json;
+use toml;
+
+pub fn read_got_layout(location: &str, format: ConfigFormat) -> Result<HashMap<String, u64>, Diagnostic> {
+    if !Path::new(location).is_file() {
+        // Assume if the file doesn't exist that there is no existing GOT layout yet. write_got_layout will handle
+        // creating our file when we want to.
+        return Ok(HashMap::new());
+    }
+
+    let s = fs::read_to_string(location)
+        .map_err(|_| Diagnostic::new("GOT layout could not be read from file"))?;
+    match format {
+        ConfigFormat::JSON => serde_json::from_str(&s)
+            .map_err(|_| Diagnostic::new("Could not deserialize GOT layout from JSON")),
+        ConfigFormat::TOML => {
+            toml::de::from_str(&s).map_err(|_| Diagnostic::new("Could not deserialize GOT layout from TOML"))
+        }
+    }
+}
+
+fn write_got_layout(
+    got_entries: HashMap<String, u64>,
+    location: &str,
+    format: ConfigFormat,
+) -> Result<(), Diagnostic> {
+    let s = match format {
+        ConfigFormat::JSON => serde_json::to_string(&got_entries)
+            .map_err(|_| Diagnostic::new("Could not serialize GOT layout to JSON"))?,
+        ConfigFormat::TOML => toml::ser::to_string(&got_entries)
+            .map_err(|_| Diagnostic::new("Could not serialize GOT layout to TOML"))?,
+    };
+
+    fs::write(location, s).map_err(|_| Diagnostic::new("GOT layout could not be written to file"))
+}
+
 ///Represents a parsed project
 ///For this struct to be built, the project would have been parsed correctly and an AST would have
 ///been generated
@@ -271,11 +309,18 @@ impl<T: SourceContainer + Sync> AnnotatedProject<T> {
     }
 
     pub fn codegen_to_string(&self, compile_options: &CompileOptions) -> Result<Vec<String>, Diagnostic> {
+        let got_layout = if let OnlineChange::Enabled { file_name, format } = &compile_options.online_change {
+            read_got_layout(file_name, *format)?
+        } else {
+            HashMap::default()
+        };
+        let got_layout = Mutex::new(got_layout);
+
         self.units
             .iter()
             .map(|AnnotatedUnit { unit, dependencies, literals }| {
                 let context = CodegenContext::create();
-                self.generate_module(&context, compile_options, unit, dependencies, literals)
+                self.generate_module(&context, compile_options, unit, dependencies, literals, &got_layout)
                     .map(|it| it.persist_to_string())
             })
             .collect()
@@ -286,11 +331,18 @@ impl<T: SourceContainer + Sync> AnnotatedProject<T> {
         context: &'ctx CodegenContext,
         compile_options: &CompileOptions,
     ) -> Result<Option<GeneratedModule<'ctx>>, Diagnostic> {
+        let got_layout = if let OnlineChange::Enabled { file_name, format } = &compile_options.online_change {
+            read_got_layout(file_name, *format)?
+        } else {
+            HashMap::default()
+        };
+        let got_layout = Mutex::new(got_layout);
+
         let Some(module) = self
             .units
             .iter()
             .map(|AnnotatedUnit { unit, dependencies, literals }| {
-                self.generate_module(context, compile_options, unit, dependencies, literals)
+                self.generate_module(context, compile_options, unit, dependencies, literals, &got_layout)
             })
             .reduce(|a, b| {
                 let a = a?;
@@ -310,13 +362,16 @@ impl<T: SourceContainer + Sync> AnnotatedProject<T> {
         unit: &CompilationUnit,
         dependencies: &FxIndexSet<Dependency>,
         literals: &StringLiterals,
+        got_layout: &Mutex<HashMap<String, u64>>,
     ) -> Result<GeneratedModule<'ctx>, Diagnostic> {
         let mut code_generator = plc::codegen::CodeGen::new(
             context,
             compile_options.root.as_deref(),
             &unit.file_name,
             compile_options.optimization,
             compile_options.debug_level,
+            //FIXME don't clone here
+            compile_options.online_change.clone(),
         );
         //Create a types codegen, this contains all the type declarations
         //Associate the index type with LLVM types
@@ -326,8 +381,9 @@ impl<T: SourceContainer + Sync> AnnotatedProject<T> {
             literals,
             dependencies,
             &self.index,
+            got_layout,
         )?;
-        code_generator.generate(context, unit, &self.annotations, &self.index, &llvm_index)
+        code_generator.generate(context, unit, &self.annotations, &self.index, llvm_index)
     }
 
     pub fn codegen_single_module<'ctx>(
@@ -372,6 +428,14 @@ impl<T: SourceContainer + Sync> AnnotatedProject<T> {
         });
         ensure_compile_dirs(targets, &compile_directory)?;
         let targets = if targets.is_empty() { &[Target::System] } else { targets };
+
+        let got_layout = if let OnlineChange::Enabled { file_name, format } = &compile_options.online_change {
+            read_got_layout(file_name, *format)?
+        } else {
+            HashMap::default()
+        };
+        let got_layout = Mutex::new(got_layout);
+
         let res = targets
             .par_iter()
             .map(|target| {
@@ -409,8 +473,15 @@ impl<T: SourceContainer + Sync> AnnotatedProject<T> {
                         };
 
                         let context = CodegenContext::create(); //Create a build location for the generated object files
-                        let module =
-                            self.generate_module(&context, compile_options, unit, dependencies, literals)?;
+                        let module = self.generate_module(
+                            &context,
+                            compile_options,
+                            unit,
+                            dependencies,
+                            literals,
+                            &got_layout,
+                        )?;
+
                         module
                             .persist(
                                 Some(&compile_directory),
@@ -429,6 +500,10 @@ impl<T: SourceContainer + Sync> AnnotatedProject<T> {
             })
             .collect::<Result<Vec<_>, Diagnostic>>()?;
 
+        if let OnlineChange::Enabled { file_name, format } = &compile_options.online_change {
+            write_got_layout(got_layout.into_inner().unwrap(), file_name, *format)?;
+        }
+
         Ok(res)
     }
 

compiler/plc_driver/src/pipelines.rs

@@ -5,7 +5,7 @@ expression: "results.join(\"\\n\")"
 ; ModuleID = 'main.st'
 source_filename = "main.st"
 
-define i16 @main() section "fn-$RUSTY$main:i16" {
+define i16 @main() {
 entry:
   %main = alloca i16, align 2
   store i16 0, i16* %main, align 2
@@ -14,17 +14,17 @@ entry:
   ret i16 %main_ret
 }
 
-declare i16 @external() section "fn-$RUSTY$external:i16"
+declare i16 @external()
 
 ; ModuleID = 'external.st'
 source_filename = "external.st"
 
-declare i16 @external() section "fn-$RUSTY$external:i16"
+declare i16 @external()
 
 ; ModuleID = '__init___TestProject'
 source_filename = "__init___TestProject"
 
-define void @__init___TestProject() section "fn-$RUSTY$__init___TestProject:v" {
+define void @__init___TestProject() {
 entry:
   ret void
 }

compiler/plc_driver/src/tests/snapshots/plc_driver__tests__external_files__external_file_function_call.snap

@@ -5,10 +5,10 @@ expression: "results.join(\"\\n\")"
 ; ModuleID = 'main.st'
 source_filename = "main.st"
 
-@x = external global i16, section "var-$RUSTY$x:i16"
-@y = external global i16, section "var-$RUSTY$y:i16"
+@x = external global i16
+@y = external global i16
 
-define i16 @main() section "fn-$RUSTY$main:i16" {
+define i16 @main() {
 entry:
   %main = alloca i16, align 2
   store i16 0, i16* %main, align 2
@@ -19,20 +19,20 @@ entry:
   ret i16 %main_ret
 }
 
-declare i16 @external() section "fn-$RUSTY$external:i16"
+declare i16 @external()
 
 ; ModuleID = 'external.st'
 source_filename = "external.st"
 
-@x = external global i16, section "var-$RUSTY$x:i16"
-@y = external global i16, section "var-$RUSTY$y:i16"
+@x = external global i16
+@y = external global i16
 
-declare i16 @external() section "fn-$RUSTY$external:i16"
+declare i16 @external()
 
 ; ModuleID = '__init___TestProject'
 source_filename = "__init___TestProject"
 
-define void @__init___TestProject() section "fn-$RUSTY$__init___TestProject:v" {
+define void @__init___TestProject() {
 entry:
   ret void
 }