Update NGCA post-processing to handle AUSPOS-v3.0

run_scripts/run_APREF_POST_PROCESSING.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/bash
 
 # APREF solution
-YYYYMMDD=20240323
+YYYYMMDD=20240713
 
 # Comments
 printf "\nAPREF solution selected: $YYYYMMDD...\n"
@@ -24,7 +24,7 @@ exciseStationsAPREF.py
 # - ITRF2020@2015.0 --> ITRF2014@2015.0
 # - Comment this out if APREF is ITRF2014
 printf "\nRunning transformSINEX.py, can take some time...\n"
-transformSINEX.py
+transformSINEX_APREF.py
 
 # Create Type-B uncertanty file
 printf "\nRunning createTypeB.py...\n"

run_scripts/run_NGCA_DOWNLOAD_FROM_AUSPOS.sh

@@ -1,13 +1,13 @@
 #!/usr/bin/bash
 
 # Archive
-ARCHIVE=20240515
+ARCHIVE=20240715
 
 # Jurisdiction
-JURIS="act"
+JURIS="qld"
 
 # IP Adress of Geodesy machine (sometimes this changes)
-IP_ADRESS=[IP ADRESS]
+IP_ADRESS=[IP ADDRESS]
 
 # Commentary
 printf "\nWill run script to copy over files after AUSPOS processing.\n"
@@ -24,23 +24,23 @@ cd /home/fedora/ngca/$JURIS
 
 # Copy over SINEX files
 printf "\nCopying over SINEX files...\n"
-scp -r geodesy@${IP_ADRESS}:/data/craig/$ARCHIVE/solutions/*.SNX sinexFiles/
+scp -i ~/.ssh/ga_ngca -r geodesy@${IP_ADRESS}:/data/craig/$ARCHIVE/solutions/*.SNX sinexFiles/
 
 # Copy over RINEX files
 printf "\nCopying over RINEX files...\n"
-scp -r geodesy@${IP_ADRESS}:/data/craig/$ARCHIVE/solutions/*_ls rinexantls/
+scp -i ~/.ssh/ga_ngca -r geodesy@${IP_ADRESS}:/data/craig/$ARCHIVE/solutions/*_ls rinexantls/
 
 # Copy over job submition log file
 printf "\nCopying over log_4_submit.txt...\n"
-scp -r geodesy@${IP_ADRESS}:/data/craig/$ARCHIVE/log_4_submit.txt $ARCHIVE
+scp -i ~/.ssh/ga_ngca -r geodesy@${IP_ADRESS}:/data/craig/$ARCHIVE/log_4_submit.txt $ARCHIVE
 
 # Copy over job dowload file
 printf "\nCopying over down_4_SNX.sh...\n"
-scp -r geodesy@${IP_ADRESS}:/data/craig/$ARCHIVE/down_4_SNX.sh $ARCHIVE
+scp -i ~/.ssh/ga_ngca -r geodesy@${IP_ADRESS}:/data/craig/$ARCHIVE/down_4_SNX.sh $ARCHIVE
 
 # Copy over nohup file
 printf "\nCopying over nohup.out...\n"
-scp -r geodesy@${IP_ADRESS}:/data/craig/$ARCHIVE/nohup.out $ARCHIVE
+scp -i ~/.ssh/ga_ngca -r geodesy@${IP_ADRESS}:/data/craig/$ARCHIVE/nohup.out $ARCHIVE
 
 
 printf "\n------ DONE ------\n"

run_scripts/run_NGCA_POST_PROCESSING.sh

@@ -1,11 +1,11 @@
 #!/usr/bin/bash
 
 # Archive
-ARCHIVE=20240515
+ARCHIVE=20240715
 
 # List of jurisdictions to loop through (largest to smallest)
-#JURIS_LIST=("act" "tas" "vic" "sa" "nt" "wa" "qld" "nsw")
-JURIS_LIST=("act" "tas" "sa")
+JURIS_LIST=("act" "tas" "vic" "sa" "nt" "wa" "qld" "nsw")
+#JURIS_LIST=("act")
 
 # Commentary 
 printf "\nWill run script to organise NGCA files after the AUSPOS processing.\n"
@@ -28,6 +28,11 @@ for JURIS in ${JURIS_LIST[*]}; do
     printf "\nExecuting nameChanges.pl...\n"
     nameChanges.pl
 
+    # Coordinate transformation
+    # - ITRF2020 --> ITRF2014
+    printf "\nExecuting transformSINEX_NGCA.py...\n"
+    transformSINEX_NGCA.py
+
     # Remove stations outside of GDA2020 from NGCA 
     printf "\nExecuting exciseStationsNGCA.py...\n"
     exciseStationsNGCA.py
@@ -80,7 +85,7 @@ for JURIS in ${JURIS_LIST[*]}; do
     printf "\nExecuting fullAdjust.py...\n"
     fullAdjust.py
 
-    # Organise results (?)
+    # Organise results
     printf "\nExecuting ngcaResults.py...\n"
     ngcaResults.py
 

run_scripts/run_NGCA_PRE_PROCESSING.sh

@@ -1,11 +1,11 @@
 #!/usr/bin/bash
 
 # Archive
-ARCHIVE=20240515
+ARCHIVE=20240715
 
 # List of jurisdictions to loop through (largest to smallest)
-JURIS_LIST=("act" "tas" "vic" "sa" "nt" "qld" "nsw")
-#JURIS_LIST=("wa")
+JURIS_LIST=("tas" "vic" "sa" "wa" "nt" "qld" "nsw")
+#JURIS_LIST=("act")
 
 # Create automatic notes file (this will remove existing file)
 rm -f /home/fedora/ngca/auto_notes_$ARCHIVE.txt

transformSINEX.py → transformSINEX_APREF.py

@@ -8,21 +8,15 @@
 # then will write out a new SINEX file. The input file is 
 # copied with .ITRF2020 suffix.
 #
+# If a TransformationSD object becomes available in GeodePy,
+# there is some grayed out code that will handle the transformation
+# of uncertainties.
+#
 # Transformation: ITRF2020@RefEpoch --> ITRF2014@RefEpoch
 #
 # Input:          AUS0OPSSNX_YYYYDDD_YYYYDDD_00U_SOL.SNX.AUS
 #
-# Output:         AUS0OPSSNX_YYYYDDD_YYYYDDD_00U_SOL.SNX.AUS
-#
-# To Do:
-#           - If a TransformationSD object becomes available in GeodePy,
-#             there is some grayed out code that will habdle the transformation
-#             of uncertainties. 
-#           - Various convenience functions were added to the gnss.py module. These
-#           - will need develpment to make them more generalised - e.g. changes in 
-#             upper/lower triangle, coordinates/velocities/, and documentation. In 
-#             addition to those, it would be udeful to investigate the performance 
-#             dataframes vs. other datatypes in the case of large SINEX files.  
+# Output:         AUS0OPSSNX_YYYYDDD_YYYYDDD_00U_SOL.SNX.AUS 
 
 ## ===
 # SETUP
@@ -223,10 +217,10 @@
 # Write SINEX 
 gnss.writeSINEX(
                 ifile,
-                snx_header,
-                snx_comments, 
-                snx_block_SiteID, 
-                snx_block_SolnEpochs, 
-                snx_block_SolnEstimate, 
-                snx_block_SolnMatrixEstimate
+                header=snx_header,
+                comment=snx_comments, 
+                siteID=snx_block_SiteID, 
+                solutionEpochs=snx_block_SolnEpochs, 
+                solutionEstimate=snx_block_SolnEstimate, 
+                solutionMatrixEstimate=snx_block_SolnMatrixEstimate
 )

transformSINEX_NGCA.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+
+## ===
+# DESCRIPTION
+# This script is designed to read in all of the SINEX files
+# from the latest NGCA archive of a particular jurisdiction
+# It performs a coordinate transformation to the solution 
+# estimates and then will write out a new SINEX file. The 
+# input file is copied with .ITRF2020 suffix.
+#
+# Transformation: ITRF2020@ObsEpoch --> ITRF2014@ObsEpoch
+#
+# Input:          [sinex_name].SNX
+#
+# Output:         [sinex_name].SNX
+#
+
+## ===
+# SETUP
+
+# Packages
+import glob
+import os
+import sys
+import pandas as pd
+import numpy as np
+from geodepy import gnss, transform, constants
+
+# Directory
+os.chdir('../sinexFiles')
+
+# Input file
+for f in glob.glob('*.SNX'):
+
+    ifile = f
+
+    # Copy original file to new name
+    new_name = ifile.split('.')[0]
+    os.system('cp ./' + ifile + ' ./' + new_name + '.ITRF2020')
+
+    ## ===
+    # READ SINEX BLOCKS
+    # - Header
+    # - FILE/REFERENCE
+    # - INPUT/ACKNOWLEDGMENTS
+    # - SOLUTION/STATISTICS
+    # - SITE/ID
+    # - SITE/RECEIVER
+    # - SITE/ANTENNA
+    # - SITE/GPS_PHASE_CENTER
+    # - SITE/ECCENTRICITY
+    # - SOLUTION/EPOCHS
+    # - SOLUTION/ESTIMATE (dataframe for computation)
+    # - SOLUTION/APRIORI (dataframe for computation)
+    # - SOLUTION/MATRIX_ESTIMATE
+    # - SOLUTION/MATRIX_APRIORI
+
+    # Header
+    snx_header = gnss.read_sinex_header_line(ifile)
+
+    # FILE/REFERENCE
+    snx_fileReference = gnss.read_sinex_file_reference_block(ifile)
+
+    # INPUT/ACKNOWLEDGMENTS
+    snx_inputAcknowledgements = gnss.read_sinex_input_acknowledgments_block(ifile)
+
+    # SOLUTION/STATISTICS
+    snx_solnStatistics = gnss.read_sinex_solution_statistics_block(ifile)
+
+    # SITE/ID
+    snx_siteID = gnss.read_sinex_site_id_block(ifile)
+
+    # SITE/RECEIVER
+    snx_siteReceiver = gnss.read_sinex_site_receiver_block(ifile)
+
+    # SITE/ANTENNA
+    snx_siteAntenna = gnss.read_sinex_site_antenna_block(ifile)
+
+    # SITE/GPS_PHASE_CENTER
+    snx_siteGpsPhaseCenter = gnss.read_sinex_site_gps_phase_center_block(ifile)
+
+    # SITE/ECCENTRICITY
+    snx_siteEccentricity = gnss.read_sinex_site_eccentricity_block(ifile)
+
+    # SOLUTION/EPOCHS
+    snx_solnEpochs = gnss.read_sinex_solution_epochs_block(ifile)
+
+    # SOLUTION/ESTIMATE 
+    # - dataframe
+    df_solnEstimate = gnss.sinex2dataframe_solution_estimate(ifile)
+
+    # SOLUTION/APRIORI
+    # - dataframe
+    df_solnApriori = gnss.sinex2dataframe_solution_apriori(ifile)
+
+    # SOLUTION/MATRIX_ESTIMATE
+    snx_block_solnMatrixEstimate = gnss.read_sinex_solution_matrix_estimate_block(ifile)
+
+    # SOLUTION/MATRIX_APRIORI
+    snx_block_solnMatrixApriori = gnss.read_sinex_solution_matrix_apriori_block(ifile)
+
+    ## ===
+    # TRANSFORM COORDINATES (Solution Estimate)
+    # - ITRF2020@ObsEpoch --> ITR2014@ObsEpoch
+
+    # Isolate coordinates
+    Xi = df_solnEstimate[df_solnEstimate['par'] == 'STAX']['est'].values
+    Yi = df_solnEstimate[df_solnEstimate['par'] == 'STAY']['est'].values
+    Zi = df_solnEstimate[df_solnEstimate['par'] == 'STAZ']['est'].values
+
+    # Transform coordinates
+    X = []
+    Y = []
+    Z = []
+    for i in range(len(Xi)):
+
+        # Coordinate transformation
+        x, y, z, vcv = transform.conform7(Xi[i], Yi[i], Zi[i], constants.itrf2020_to_itrf2014)
+
+        # Append to list
+        X.append(x)
+        Y.append(y)
+        Z.append(z)
+
+    ## ===
+    # REPLACE DATAFRAME ESTIMATES (Solution Estimate)
+    # - Coordinates (X,Y,Z)
+
+    # Coordinates
+    i = 0
+    k = 0
+    j = 0
+    for r in range(len(df_solnEstimate.code)):
+
+        # Replace X estimate
+        if df_solnEstimate.loc[r, "par"] == "STAX":
+            df_solnEstimate.loc[r, "est"] = X[i]
+            i += 1
+
+        # Replace Y estimate
+        if df_solnEstimate.loc[r, "par"] == "STAY":
+            df_solnEstimate.loc[r, "est"] = Y[k]
+            k += 1
+
+        # Replace Z estimate
+        if df_solnEstimate.loc[r, "par"] == "STAZ":
+            df_solnEstimate.loc[r, "est"] = Z[j]
+            j += 1
+
+    # Write to sinex format
+    snx_block_solnEstimate = gnss.dataframe2sinex_solution_estimate(df_solnEstimate)
+
+    ## ===
+    # TRANSFORM COORDINATES (Solution Apriori)
+    # - ITRF2020@ObsEpoch --> ITR2014@ObsEpoch
+
+    # Isolate coordinates
+    Xi = df_solnApriori[df_solnApriori['par'] == 'STAX']['est'].values
+    Yi = df_solnApriori[df_solnApriori['par'] == 'STAY']['est'].values
+    Zi = df_solnApriori[df_solnApriori['par'] == 'STAZ']['est'].values
+
+    # Transform coordinates
+    X = []
+    Y = []
+    Z = []
+    for i in range(len(Xi)):
+
+        # Coordinate transformation
+        x, y, z, vcv = transform.conform7(Xi[i], Yi[i], Zi[i], constants.itrf2020_to_itrf2014)
+
+        # Append to list
+        X.append(x)
+        Y.append(y)
+        Z.append(z)
+
+    ## ===
+    # REPLACE DATAFRAME ESTIMATES (Solution Apriori)
+    # - Coordinates (X,Y,Z)
+
+    # Coordinates
+    i = 0
+    k = 0
+    j = 0
+    for r in range(len(df_solnApriori.code)):
+
+        # Replace X estimate
+        if df_solnApriori.loc[r, "par"] == "STAX":
+            df_solnApriori.loc[r, "est"] = X[i]
+            i += 1
+
+        # Replace Y estimate
+        if df_solnApriori.loc[r, "par"] == "STAY":
+            df_solnApriori.loc[r, "est"] = Y[k]
+            k += 1
+
+        # Replace Z estimate
+        if df_solnApriori.loc[r, "par"] == "STAZ":
+            df_solnApriori.loc[r, "est"] = Z[j]
+            j += 1
+
+    # Write to sinex format
+    snx_block_solnApriori = gnss.dataframe2sinex_solution_apriori(df_solnApriori)
+
+    ## ===
+    # WRITE TO SINEX FILE
+
+    # Write SINEX 
+    gnss.writeSINEX(
+                    ifile,
+                    header=snx_header,
+                    fileReference=snx_fileReference, 
+                    inputAcknowledgments=snx_inputAcknowledgements, 
+                    solutionStatistics=snx_solnStatistics,
+                    siteID=snx_siteID,
+                    siteReceiver=snx_siteReceiver,
+                    siteAntenna=snx_siteAntenna,
+                    siteGpsPhaseCenter=snx_siteGpsPhaseCenter,
+                    siteEccentricity=snx_siteEccentricity,
+                    solutionEpochs=snx_solnEpochs,
+                    solutionEstimate=snx_block_solnEstimate,
+                    solutionApriori=snx_block_solnApriori, 
+                    solutionMatrixEstimate=snx_block_solnMatrixEstimate,
+                    solutionMatrixApriori=snx_block_solnMatrixApriori,
+    )
+
+# Remove ITRF2020 SINEX files
+for f in glob.glob('*.ITRF2020'):
+    os.remove(f)