Update for MQ 9.3.2

Author: ibmmqmet

CHANGES.md

@@ -2,6 +2,12 @@
 History (newest at top)
 =======================
 
+Feb 2023 (v5.4.2)
+* Update for MQ V9.3.2
+* WQ structure extended with streamed message count
+* Fix problem when final record in input data was truncated/corrupt
+* Removed QIST RAIO/RABP fields as they are never filled in by qmgr so misleading
+
 Oct 2022 (v5.4.1)
 * Update for MQ V9.3.1
 * Extended QQST structure in the 115 records

bin/aix/convH

@@ -4,7 +4,7 @@ plat=`uname`
 
 if [ -z "$VERS" ]
 then
-  VERS="931"
+  VERS="932"
 fi
 
 if [ "$plat" = "AIX" ]

bin/aix/mqsmfcsv

@@ -3,7 +3,7 @@ CC=i686-w64-mingw32-gcc
 # CFLAGS= -I. -m32 -fpack-struct=8 -DPLATFORM_WINDOWS 
 CFLAGS= -I. -m32  -DPLATFORM_WINDOWS 
 PLATFLAGS=
-VERS=931
+VERS=932
 SRC = mqsmf.c \
       smfDDL.c \
       smfDate.c \

bin/linux/convH

@@ -1,6 +1,6 @@
 
 CC=cc
-CFLAGS= -I.
+CFLAGS= -I. -O2
 PLATFLAGS=
 SRC = \
       smfDDL.c \

bin/linux/mqsmfcsv

@@ -1,5 +1,5 @@
 CFLAGS=-nologo /D_CRT_SECURE_NO_WARNINGS /Zp1 /J /O2 /DPLATFORM_WINDOWS
-VERS=931
+VERS=932
 SRC = mqsmf.c \
       smfDDL.c \
       smfDate.c \

bin/win/convH.exe

@@ -26,7 +26,7 @@
  * to an explicitly-sized standard datatype (eg uint64). This
  * is needed if the program is compiled in 64-bit mode. We also
  * rewrite bitfield elements in structures.
- * 
+ *
  * Two variants of the output are created, one for Windows
  * and one for Unix, based on the PLATFORM_* definitions because
  * of different paddings done for bitfields.
@@ -105,6 +105,7 @@ int inBitfieldReplace = 0;
 void bitfieldReprint(int c);
 void datatypeReplace(char *);
 char *stripLine(char *);
+char *trimLine(char *);
 void printHeader(char *);
 
 int main(int argc, char **argv)
@@ -140,6 +141,8 @@ int main(int argc, char **argv)
      c = fgets(line,sizeof(line)-1,stdin);
      if (c)
      {
+       c = trimLine(line);
+
        if (strstr(line,"pragma"))
          continue;
        /****************************************************/
@@ -185,6 +188,26 @@ int main(int argc, char **argv)
   return 0;
 }
 
+/* Strip trailing whitespace but leave the final "\n" alone */
+char *trimLine(char *l) {
+  char *p;
+  int len;
+  int trimmed = 0;
+  len = strlen(l);
+  if (len > 2) {
+    p = &l[len-2]; /* Start before the final '\n' */
+    while (*p == ' ' || *p == '\t') {
+      trimmed=1;
+      *p=0;
+      p--;
+    }
+    if (trimmed) {
+      strcat(l,"\n"); /* Put the trailing CR back */
+    }
+  }
+  return l;
+}
+
 /* Remove trailing comments and leading spaces */
 char *stripLine(char *l) {
   char *p = l;
@@ -198,7 +221,7 @@ char *stripLine(char *l) {
   cmtEnd  = strstr(p,"*/\n");
 
   /* There are a few multi-line macros that have comments embedded
-   * so we can't just strip to end of line ... if the line ends in 
+   * so we can't just strip to end of line ... if the line ends in
    * a continuation marker then leave the comments alone
    */
   if (cmtStart != NULL && cmtEnd != NULL && p[strlen(p)-2] != '\\') {
@@ -389,7 +412,7 @@ void datatypeReplace(char *line) {
   return;
 }
 
-void printHeader(char *version) 
+void printHeader(char *version)
 {
   printf("/*\n");
   printf(" * This file was generated from cqsdsmfc.h version %s\n",version);

bin/win/mqsmfcsv.exe

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016,2020 IBM Corporation and other Contributors.
+ * Copyright (c) 2016,2023 IBM Corporation and other Contributors.
  *
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
@@ -163,6 +163,7 @@ static int offsetCorrection = 0;
 static BOOL warn115_240=FALSE;
 
 static time_t startTime = 0;
+static time_t endTime = 0;
 static char *formatRate;
 
 static int amsType = SMFTYPE_MQ_AMS;
@@ -207,6 +208,7 @@ int main( int argc, char *argv[] )
   int i,j;                             /* loop counters                    */
   char  dataBuf[MAX_SMF_DATA];         /* Contains the SMF data            */
   int   bytesRead;                     /* Number of bytes from fread()     */
+  int   tmpBytes;
   int   offset;                        /* total number of bytes in a record*/
   unsigned int d[3];                   /* used in date conversion          */
   unsigned int ddd,year;               /* day number and year number       */
@@ -399,8 +401,7 @@ int main( int argc, char *argv[] )
 
   fstat(fileno(fp),&statbuf);
   totalFileSize = statbuf.st_size;
-  if (debugLevel >=3 )
-    fprintf(infoStream,"Total File Size = %lld\n",totalFileSize);
+  debugf(3,"Total File Size = %lld\n",totalFileSize);
 
   convInit();      /* Decide whether this is a big or little endian machine*/
 
@@ -466,15 +467,22 @@ int main( int argc, char *argv[] )
       /* same as the SMF record length). It comes from the first two      */
       /* bytes of the Record Descriptor Word (RDW).                       */
       /********************************************************************/
-
       bytesRead = fread(&pSMFMQRecord->Header.SMFLEN,1,2,fp);
       if (bytesRead < 2)
-        continue;
+      {
+        debugf(1,"At end of input data - fread (phase 1a) returned %d bytes\n",tmpBytes);
+        break;
+      }
 
       /********************************************************************/
       /* The second half-word is the segment indicator.                   */
       /********************************************************************/
-      fread(&pSMFMQRecord->Header.SMFSEG ,1,2,fp);
+      tmpBytes = fread(&pSMFMQRecord->Header.SMFSEG ,1,2,fp);
+      if (tmpBytes < 2)
+      {
+        debugf(1,"At end of input data - fread (phase 1b) returned %d bytes\n",tmpBytes);
+        break;
+      }
 
       /********************************************************************/
       /* And then read the actual data, starting at the RECFLG field in   */
@@ -483,8 +491,7 @@ int main( int argc, char *argv[] )
       /* amount.                                                          */
       /********************************************************************/
       nextLength = conv16(pSMFMQRecord->Header.SMFLEN) - 4;
-      if (debugLevel >=3)
-        fprintf(infoStream,"   NextLen = %d bytes \n",nextLength);
+      debugf(3,"   NextLen = %d bytes \n",(nextLength));
 
       bytesRead = fread(&pSMFMQRecord->Header.SMFRECFLG, 1, nextLength , fp);
       if (bytesRead != nextLength)
@@ -504,18 +511,28 @@ int main( int argc, char *argv[] )
       /* segmentation, no SMF record is meant to be >32768 bytes, so we   */
       /* need to check that.  Once again, ignore the RDW when working     */
       /* out how much real data there is.                                 */
+      /* We've also seen input files which were truncated such that       */
+      /* the continuation records were not captured. So check that we are */
+      /* actually able to read the data and that we're not at the EOF.    */
       /********************************************************************/
       if (pSMFMQRecord->Header.SMFSEG[0] != 0)
       {
         do
         {
+          tmpBytes = fread(&nextLength ,1,2,fp);
+          if (tmpBytes < 2) {
+            debugf(1,"At end of input data - fread (phase 2a) returned %d bytes\n",tmpBytes);
+            goto mod_exit;
+          }
 
-          fread(&nextLength ,1,2,fp);
           nextLength = conv16(nextLength);
-          fread(&pSMFMQRecord->Header.SMFSEG ,1,2,fp);
+          tmpBytes = fread(&pSMFMQRecord->Header.SMFSEG ,1,2,fp);
+          if (tmpBytes < 2) {
+            debugf(1,"At end of input data - fread (phase 2b) returned %d bytes\n",tmpBytes);
+            goto mod_exit;
+          }
 
-          if (debugLevel >=3)
-            fprintf(infoStream,"   NextLen = %d bytes \n",nextLength);
+          debugf(3,"   NextLen = %d bytes \n",nextLength);
 
           if (offset+nextLength > sizeof(dataBuf))
           {
@@ -524,6 +541,10 @@ int main( int argc, char *argv[] )
           }
 
           bytesRead = fread(&dataBuf[offset], 1, nextLength-4 , fp );
+          if (bytesRead < (nextLength - 4)) {
+            debugf(1,"At end of input data - fread (phase 2c) returned %d bytes\n",tmpBytes);
+            goto mod_exit;
+          }
           offset += bytesRead;
         } while (pSMFMQRecord->Header.SMFSEG[0] != 0x02);/* end of record indicator*/
 
@@ -685,7 +706,7 @@ int main( int argc, char *argv[] )
         convDate(pqwhs->qwhsstck,dt);
         strcpy(commonF.stckFormatDate,dt[0]);
         strcpy(commonF.stckFormatDate,dt[1]);
-        if (recordType == SMFTYPE_MQ_STAT) 
+        if (recordType == SMFTYPE_MQ_STAT)
         {
           if (conv16(pqwhs->qwhslen) >= 52)
           {
@@ -745,8 +766,8 @@ int main( int argc, char *argv[] )
       }
     }
 
-    if (debugLevel >=3 && pqwhs != NULL)
-       fprintf(infoStream,"Section count %d for %4.4s, qwhslen=%d\n",sectionCount,commonF.qMgr,conv16(pqwhs->qwhslen));
+    if (pqwhs != NULL)
+       debugf(3,"Section count %d for %4.4s, qwhslen=%d\n",sectionCount,commonF.qMgr,conv16(pqwhs->qwhslen));
 
     /*********************************************************************/
     /* Once we know how many sections there are, copy the triplet values */
@@ -792,7 +813,7 @@ int main( int argc, char *argv[] )
     case SMFTYPE_MQ_STAT:
       switch(recordSubType)
       {
-      case SMFSUBTYPE_MQ_STAT_BASIC: 
+      case SMFSUBTYPE_MQ_STAT_BASIC:
         /*****************************************************************/
         /* Some of the triplets seem to refer to internal/undocumented   */
         /* structures. Only the QSST and QJST are known elements.        */
@@ -935,15 +956,15 @@ int main( int argc, char *argv[] )
 
       case SMFSUBTYPE_MQ_STAT_RESERVED_1:
         /* These are internal undocumented structures. There's not much we can do */
-        /* except ignore them. We will see them show up in the overall subtype    */ 
+        /* except ignore them. We will see them show up in the overall subtype    */
         /* stats of records processed, but no other debug.                        */
-        if (!warn115_240) 
+        if (!warn115_240)
         {
           sprintf(tmpHead,"Internal SMF %d subtype %d records found",SMFTYPE_MQ_STAT,recordSubType);
           fprintf(infoStream,"%s\n",tmpHead);
           warn115_240=TRUE;
         }
-        break; 
+        break;
 
       default:
         knownSubType = FALSE;
@@ -1163,22 +1184,20 @@ int main( int argc, char *argv[] )
   /* Cleanup and exit. If we get here normally, then the checkpoint      */
   /* file is not needed any more, so it is deleted.                      */
   /***********************************************************************/
-  if (debugLevel >= 1) {
-    fprintf(infoStream,"Removing checkpoint file\n");
-  }
+  debugf(1,"Removing checkpoint file %s\n",checkPointFileName);
   remove(checkPointFileName);
   pos = ftello(fp);
 
+mod_exit:
   if (outputFormat == OF_SQL && ddlTemplateClose != NULL) {
        closeFinalDDL();
   }
 
-mod_exit:
-
   fflush(NULL); /* Ensure all streams flushed if possible */
 
   for (i=0;i<MAXCP;i++) {
     if (checkPoint[i].fp) {
+      debugf(1,"  Closing %s\n",checkPoint[i].name);
       fclose(checkPoint[i].fp);
     }
   }
@@ -1188,8 +1207,8 @@ int main( int argc, char *argv[] )
   }
 
   formatRate = getFormatRate(pos);
-  fprintf(infoStream,"Processed %u records total %s\n",totalRecords,formatRate);
-
+  endTime = time(NULL);
+  fprintf(infoStream,"Processed %u records total %s in %d seconds \n",totalRecords,formatRate,(int)(endTime-startTime));
 
   /**********************************************************************/
   /* If the program has been restarted via checkpoint, these counts are */
@@ -1375,9 +1394,7 @@ myoff_t readCheckPoint(FILE *fp) {
   sscanf(line,"%d %lld",&startingRecords,&inputOffset);
   fprintf(infoStream,"Reading from recovery checkpoint.\n");
 
-  if (debugLevel >=1) {
-    fprintf(infoStream,"InputOffset: %lld Processed records: %d\n",inputOffset, startingRecords);
-  }
+  debugf(1,"InputOffset: %lld Processed records: %d\n",inputOffset, startingRecords);
 
   /************************************************************************/
   /* Subsequent lines have two fields naming the output structure and how */
@@ -1392,9 +1409,7 @@ myoff_t readCheckPoint(FILE *fp) {
       sscanf(line,"%s %lld",field,&offset);
       checkPoint[i].name = strdup(field);
       checkPoint[i].offset = offset;
-      if (debugLevel >=2) {
-        fprintf(infoStream,"Loaded status[%d] for %s as offset %lld\n",i,field,offset);
-      }
+      debugf(2,"Loaded status[%d] for %s as offset %lld\n",i,field,offset);
       i++;
     }
   } while (c);
@@ -1552,7 +1567,7 @@ int laterThan(int v)
     strncpy(mqVer,commonF.mqVer,4);
     currentVersion = atoi(mqVer);
   }
-  if (currentVersion > v) 
+  if (currentVersion > v)
     return 1;
   return rc;
 }

src/M

@@ -258,6 +258,10 @@ extern void   calcYMD(int y,int dY,int *m,int *dM);
 extern FILE * fopencsv(const char *, BOOL *);
 extern FILE * fopenext(const char *, const char *, BOOL *);
 
+#if !defined(debugf)
+#define debugf(_level,_fmt,...) \
+      {if (debugLevel >= _level) fprintf(infoStream,_fmt,__VA_ARGS__);}
+#endif
 /*
  * Convert some MQI values into the corresponding string
  */

src/Makefile.gcc.win

@@ -1,5 +1,5 @@
 /*
- * This file was generated from cqsdsmfc.h version 931
+ * This file was generated from cqsdsmfc.h version 932
  * by the convH program (convH64.c) in the src directory
  * Do not try to edit this header manually.
  * 
@@ -1087,17 +1087,21 @@ used during close of a topic
 uint32_t  putddly;   /* Number of messages put with
 delivery delay, and hence sent to
 delivery delay staging queue    */
+uint32_t  z__004[1];
+uint32_t  streamedn;
+uint32_t  z__005[1];
 } wq;
 #define WQ_HEXID 0xF702
 #define WQ_EYE   "WQST"
-#define WQ_VER   8
+#define WQ_VER   9
 #define WQ_VER_1 1
 #define WQ_VER_2 2
 #define WQ_VER_3 3
 #define WQ_VER_4 4
 #define WQ_VER_5 5
 #define WQ_VER_6 6
 #define WQ_VER_8 8
+#define WQ_VER_9 9
 #define WQ_LEN_1 576
 #define WQ_LEN_2 592
 #define WQ_LEN_3 608