Merge pull request #258 from sy-c/master

v2.20.0

doc/configurationParameters.md

@@ -2,7 +2,7 @@
 
 This document describes all the configuration parameters that can be set in readout.
 
-Some parameters control the behavior of readout as soon as it starts, and valid for the lifetime of the process. (1)
+Some parameters control the behavior of readout as soon as it starts, and are valid for the lifetime of the process. (1)
 
 Other parameters (2) are set whenever the control state-machine receives the CONFIGURE command (or shortly after startup when running o2-readout-exe outside from the ECS workflow, typically from the command line) and until the RESET command.
 The corresponding parameters usually come from a local file or from a central reposity, as provided by the O2 system.
@@ -11,20 +11,20 @@ They can be reloaded several time during the lifetime of the process.
 
 # Readout defaults (1)
 
-These are defined in local file `/etc/o2.d/readout-default.cfg`, which is loaded on `o2-readout-exe` startup (if file exists). They are valid for the lifetime of the o2-readout-exe process. There is usually no need for users to edit these settings. The file format is '.ini' style, with `[sections]` and `key=value` pairs.
+These are defined in local file `/etc/o2.d/readout-default.cfg`, which is loaded on `o2-readout-exe` startup (if the file exists). They are valid for the lifetime of the o2-readout-exe process. There is usually no need for users to edit these settings. The file format is '.ini' style, with `[sections]` and `key=value` pairs.
 
-Following table describe the parameters of the `[readout]` section.
+Following table describes the parameters of the `[readout]` section.
 
 | Parameter name  | Type | Default value | Description |
 |--|--|--|--|
 | memLock | int | 0 | Defines the memory swapping policy. If set, all readout memory is locked in RAM with mlockall() |
 | readoutExe | string | | If set, overwrites the current running executable with the one at given path. Used for tests. |
 | readoutConfig | string | | If set with readoutExe, launch the new exe with this parameter as argument. Used for tests. |
-| statsPublishAddress | string | | If set, address where to publish readout stats, eg "tcp://127.0.0.1:6008". Connect to this service with o2-readout-monitor. |
+| statsPublishAddress | string | | If set, address where to publish readout stats, eg "tcp://127.0.0.1:6008" (o2-readout-monitor should listen there). |
 | statsPublishInterval | string | | If set, interval for readout stats publish, in seconds. |
 | db | string | | If set, defines connection parameters to a MySQL database in the form 'user:password@host/dbname'. Runtime statistics are stored in a table. Structure can be created with o2-readout-admin-db. |
 | customCommandsEnabled | int | 0 | If set, custom commands are enabled, i.e. launching external scripts at some specific state transitions. See customCommands keyword below. |
-| membanksMonitorPath | string | | Path to memory banks monitor output (a local file FIFO, to be connected with o2-readout-monitor). |
+| membanksMonitorPath | string | | Path to memory banks monitor output (a local file FIFO, to be connected with o2-readout-monitor-memory). |
 | membanksMonitorRate | double | 0 | Rate (in Hertz) at which to publish memory status. |
 
 

doc/releaseNotes.md

@@ -564,7 +564,9 @@ This file describes the main feature changes for each readout.exe released versi
 - Added protection against unhandled exceptions (e.g. from ReadoutCard).
 - Added protection against unexpected state machine transitions (e.g. ECS sending RESET while STARTING).
 
-## next version
+## v2.20.0 - 12/06/2023
 - Added a separate thread to call bookkeeping API functions, so that data flow is not affected in case they are blocking.
 - o2-readout-test-lib-bookkeeping: more options.
 - Added o2-readout-monitor-memory: to view in real time state of data pages.
+- Added consistency check of orbit vs timestamp when large gas in TF ids detected.
+- Updated configuration parameters documentation.

src/DataBlock.h

@@ -62,6 +62,7 @@ struct DataBlockHeader {
   uint32_t timeframeOrbitLast;  ///< from timeframe
   uint8_t flagEndOfTimeframe;   ///< flag to signal this is the last TF block
   uint8_t isRdhFormat;          ///< flag set when payload is RDH-formatted
+  uint32_t orbitFirstInBlock;   ///< the first orbit in this block
   uint32_t orbitOffset;         ///< set when RDH orbits should be added given offset to match TFid
   uint32_t memorySize;          ///< size in memory of current block
 
@@ -70,10 +71,10 @@ struct DataBlockHeader {
 
 // Version of this header
 // with DB marker for DataBlock start, 1st byte in header little-endian
-const uint32_t DataBlockVersion = 0x0004DBDB;
+const uint32_t DataBlockVersion = 0x0005DBDB;
 
 // DataBlockHeader instance with all default fields
-const DataBlockHeader defaultDataBlockHeader = { .headerVersion = DataBlockVersion, .headerSize = sizeof(DataBlockHeader), .dataSize = 0, .blockId = undefinedBlockId, .pipelineId = undefinedBlockId, .timeframeId = undefinedTimeframeId, .runNumber = undefinedRunNumber, .systemId = undefinedSystemId, .feeId = undefinedFeeId, .equipmentId = undefinedEquipmentId, .linkId = undefinedLinkId, .timeframeOrbitFirst = undefinedOrbit, .timeframeOrbitLast = undefinedOrbit, .flagEndOfTimeframe = 0, .isRdhFormat = 1, .orbitOffset = undefinedOrbit, .memorySize = 0, .userSpace = { 0 } };
+const DataBlockHeader defaultDataBlockHeader = { .headerVersion = DataBlockVersion, .headerSize = sizeof(DataBlockHeader), .dataSize = 0, .blockId = undefinedBlockId, .pipelineId = undefinedBlockId, .timeframeId = undefinedTimeframeId, .runNumber = undefinedRunNumber, .systemId = undefinedSystemId, .feeId = undefinedFeeId, .equipmentId = undefinedEquipmentId, .linkId = undefinedLinkId, .timeframeOrbitFirst = undefinedOrbit, .timeframeOrbitLast = undefinedOrbit, .flagEndOfTimeframe = 0, .isRdhFormat = 1, .orbitFirstInBlock = undefinedOrbit, .orbitOffset = undefinedOrbit, .memorySize = 0, .userSpace = { 0 } };
 
 // DataBlock
 // Pair of header + payload data

src/ReadoutEquipment.cxx

@@ -487,12 +487,20 @@ Thread::CallbackResult ReadoutEquipment::threadCallback(void* arg)
 	}
 
         static InfoLogger::AutoMuteToken logTFdiscontinuityToken(LogWarningSupport_(3004), 10, 60);
+        static InfoLogger::AutoMuteToken logTFdiscontinuityTokenError(LogErrorSupport_(3004), 10, 60);
 
 	ptr->statsNumberOfTimeframes++;
 	// detect gaps in TF id continuity
 	if (tfId != ptr->lastTimeframe + 1) {
 	  if (ptr->cfgRdhDumpWarningEnabled) {
             theLog.log(logTFdiscontinuityToken, "Non-contiguous timeframe IDs %llu ... %llu", (unsigned long long)ptr->lastTimeframe, (unsigned long long)tfId);
+            // check if difference is large and orbit consistant with timestamp
+            double now = ptr->firstTimeframeTimestamp.getTime();
+            double dt = (nextBlock->getData()->header.orbitFirstInBlock - ptr->firstTimeframeHbOrbitBegin) * 1.0 / ptr->LHCOrbitRate; // diff in orbit / orbit rate = should be close to current timestamp
+            uint32_t expected = ptr->firstTimeframeHbOrbitBegin + (uint32_t)(now * ptr->LHCOrbitRate);
+            if (fabs(dt - now) > 10) {
+              theLog.log(logTFdiscontinuityTokenError, "Orbit 0x%X seems inconsistent from expected ~0x%X (orbit rate %u, elapsed time %.1fs)", (int)nextBlock->getData()->header.orbitFirstInBlock, expected, ptr->LHCOrbitRate, now);
+            }
 	  }
 	}
 	ptr->lastTimeframe = tfId;
@@ -640,6 +648,7 @@ uint64_t ReadoutEquipment::getTimeframeFromOrbit(uint32_t hbOrbit)
 {
   if (!isDefinedFirstTimeframeHbOrbitBegin) {
     firstTimeframeHbOrbitBegin = hbOrbit;
+    firstTimeframeTimestamp.reset();
     isDefinedFirstTimeframeHbOrbitBegin = 1;
     bool isOk = true;
     gReadoutStats.mutex.lock();
@@ -730,6 +739,7 @@ int ReadoutEquipment::tagDatablockFromRdh(RdhHandle& h, DataBlockHeader& bh)
     }
   }
   getTimeframeOrbitRange(tfId, bh.timeframeOrbitFirst, bh.timeframeOrbitLast);
+  bh.orbitFirstInBlock = hbOrbit;
   bh.timeframeOrbitFirst -= bh.orbitOffset;
   bh.timeframeOrbitLast -= bh.orbitOffset;
   // printf("TF %d eq %d link %d : orbits %X - %X\n", (int)bh.timeframeId, (int)bh.equipmentId, (int)bh.linkId, (int)bh.timeframeOrbitFirst, (int)bh.timeframeOrbitLast);

src/ReadoutEquipment.h

@@ -156,6 +156,7 @@ class ReadoutEquipment
   unsigned long long statsNumberOfTimeframes = 0; // number of timeframes read out
   uint32_t firstTimeframeHbOrbitBegin = 0;        // HbOrbit of beginning of first timeframe
   bool isDefinedFirstTimeframeHbOrbitBegin = 0;
+  AliceO2::Common::Timer firstTimeframeTimestamp; // timestamp of first timeframe/orbit received, for consistency checks
 
   AliceO2::Common::Timer timeframeClock; // timeframe id should be increased at each clock cycle
   uint64_t currentTimeframe = 0;         // id of current timeframe

src/ReadoutEquipmentRORC.cxx

@@ -232,7 +232,7 @@ Thread::CallbackResult ReadoutEquipmentRORC::prepareBlocks()
       // check CRU FIFO status - but only after first loop, otherwise would be empty yet
       if (!channel->areSuperpageFifosHealthy()) {
 	static InfoLogger::AutoMuteToken logToken(LogWarningSupport_(3235), 5, 60);
-	theLog.log(logToken, "Equipment %s: ROC memory fifo not healthy", name.c_str());
+	theLog.log(logToken, "Equipment %s: ROC is lacking data pages", name.c_str());
       }
     }
   }

src/ReadoutVersion.h

@@ -9,5 +9,5 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
-#define READOUT_VERSION "2.19.1"
+#define READOUT_VERSION "2.20.0"