Exercise 1

Author: anosh-ar

exercises/Airports.sqlite

@@ -0,0 +1,106 @@
+// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+
+// Example 1:  Airports
+// Learning goals:
+// - Understand the core concepts pipeline, block, and pipe
+// - Understand the general structure of a pipeline
+
+// 1. This Jayvee model describes a pipeline 
+// from a CSV file in the web 
+// to a SQLite file sink.
+pipeline AirportsPipeline {
+
+    // 2. We describe the structure of the pipeline,
+    // usually at the top of the pipeline.
+    // by connecting blocks via pipes. 
+
+    // 3. Syntax of a pipe
+    // connecting the block  AirportsExtractor
+    // with the block  AirportsTextFileInterpreter.
+    AirportsExtractor -> AirportsTextFileInterpreter;
+
+    // 4. The output of the preceding block is hereby used 
+    // as input for the succeeding block.
+
+    // 5. Pipes can be further chained, 
+    // leading to an overview of the pipeline.
+     AirportsTextFileInterpreter
+        -> AirportsCSVInterpreter 
+        //-> NameHeaderWriter
+        -> AirportsTableInterpreter
+        -> AirportsLoader;
+
+
+    // 6. Below the pipes, we usually define the blocks 
+    // that are connected by the pipes.
+
+    // 7. Blocks instantiate a blocktype by using the oftype keyword.
+    // The blocktype defines the available properties that the block
+    // can use to specify the intended behavior of the block 
+    block AirportsExtractor oftype HttpExtractor {
+
+        // 8. Properties are assigned to concrete values.
+        // Here, we specify the URL where the file shall be downloaded from.
+        url: "https://opendata.rhein-kreis-neuss.de/api/explore/v2.1/catalog/datasets/rhein-kreis-neuss-flughafen-weltweit/exports/csv?lang=en&timezone=Europe%2FBerlin&use_labels=true&delimiter=%3B";
+    }
+
+    // 9. The HttpExtractor requires no input and produces a binary file as output.
+    // This file has to be interpreted, e.g., as text file.
+    block AirportsTextFileInterpreter oftype TextFileInterpreter { }
+
+    // 10. Next, we interpret the text file as sheet.
+    // A sheet only contains text cells and is useful for manipulating the shape of data before assigning more strict value types to cells.
+    block AirportsCSVInterpreter oftype CSVInterpreter {
+        //enclosing: '"';
+        delimiter: ";";
+    }
+
+    // 11. We can write into cells of a sheet using the CellWriter blocktype.
+    block NameHeaderWriter oftype CellWriter {
+        // 12. We utilize a syntax similar to spreadsheet programs.
+        // Cell ranges can be described using the keywords "cell", "row", "column", or "range" that indicate which 
+        // cells are selected for the write action.
+        at: cell A1;
+
+        // 13. For each cell we selected with the "at" property above,
+        // we can specify what value shall be written into the cell.
+        write: ["name"];
+    }
+
+    // 14. As a next step, we interpret the sheet as a table by adding structure.
+    // We define a valuetype per column that specifies the data type of the column.
+    // Rows that include values that are not valid according to the their valuetypes are dropped automatically. 
+    block AirportsTableInterpreter oftype TableInterpreter {
+        header: true;
+        columns: [
+            "Lfd. Nummer" oftype integer,
+            "Name des Flughafens" oftype text,
+            "Ort" oftype text,
+            "Land" oftype text,
+            "IATA" oftype text,
+            "ICAO" oftype text,
+            "Latitude" oftype decimal,
+            "Longitude" oftype decimal,
+            "Altitude" oftype decimal,
+            "Zeitzone" oftype decimal,
+            "DST" oftype text,
+            "Zeitzonen-Datenbank" oftype text,
+            "geo_punkt" oftype text
+        ];
+    }
+
+    // 15. As a last step, we load the table into a sink,
+    // here into a sqlite file.
+    // The structural information of the table is used
+    // to generate the correct table.
+    block AirportsLoader oftype SQLiteLoader {
+        table: "Airports";
+        file: "./Airports.sqlite";
+    }
+
+    // 16. Congratulations!
+    // You can now use the sink for your data analysis, app, 
+    // or whatever you want to do with the cleaned data.  
+}