billing: Add option for json format

Motivation:

See GitHub issue dCache#7421 dCache#7421

Modification:

The json format strings and an option to use this format was added to the
properties-file of billing. If the json option is set to true, the json
format will be used instead of text format.

Result:

Billing files can be written in json format.

Acked-by:
Target: master
Require-book: yes
Require-notes: yes

docs/TheBook/src/main/markdown/config-billing.md

@@ -25,6 +25,7 @@ If you installed dCache following the instructions in the Chapter [Installing dC
 ```
 
 Use the property `billing.text.dir` to set the location of the log files and the property `billing.enable.text` to control whether the plain-text log files are generated.
+To write the logs in JSON format instead of plain text, use the property `billing.format.json`.
 
 By default the log files are located in the directory
 `/var/lib/dcache/billing`. Under this directory the log files are

modules/dcache/src/main/java/org/dcache/services/billing/cells/BillingCell.java

@@ -63,7 +63,8 @@ public final class BillingCell
 
     private static final Logger LOGGER =
           LoggerFactory.getLogger(BillingCell.class);
-    public static final String FORMAT_PREFIX = "billing.text.format.";
+    public static final String TEXT_FORMAT_PREFIX = "billing.text.format.";
+    public static final String JSON_FORMAT_PREFIX = "billing.json.format.";
 
     private final SimpleDateFormat _fileNameFormat =
           new SimpleDateFormat("yyyy.MM.dd");
@@ -87,6 +88,7 @@ public final class BillingCell
     private CellStub _poolManagerStub;
     private Path _logsDir;
     private boolean _enableText;
+    private boolean _jsonFormat;
     private boolean _flatTextDir;
 
     public BillingCell() {
@@ -102,10 +104,10 @@ public void setEnvironment(final Map<String, Object> environment) {
         };
         for (Map.Entry<String, Object> e : environment.entrySet()) {
             String key = e.getKey();
-            if (key.startsWith(FORMAT_PREFIX)) {
+            if (_jsonFormat ? key.startsWith(JSON_FORMAT_PREFIX) : key.startsWith(TEXT_FORMAT_PREFIX)) {
                 String format = Formats.replaceKeywords(String.valueOf(e.getValue()), replaceable);
                 String clazz = CaseFormat.LOWER_HYPHEN.to(CaseFormat.UPPER_CAMEL,
-                      key.substring(FORMAT_PREFIX.length()));
+                      key.substring(_jsonFormat ? JSON_FORMAT_PREFIX.length() : TEXT_FORMAT_PREFIX.length()));
                 _formats.put(clazz, format);
             }
         }
@@ -450,4 +452,8 @@ public void setEnableTxt(boolean enableText) {
         _enableText = enableText;
     }
 
+    public void setJsonFormat(boolean jsonFormat) {
+        _jsonFormat = jsonFormat;
+    }
+
 }

modules/dcache/src/main/resources/org/dcache/services/billing/cells/billing.xml

@@ -39,6 +39,7 @@
     class="org.dcache.services.billing.cells.BillingCell">
     <property name="logsDir" value="${billing.text.dir}"/>
     <property name="enableTxt" value="${billing.enable.text}"/>
+    <property name="jsonFormat" value="${billing.format.json}"/>
     <property name="flatTextDir" value="${billing.text.flat-dir}"/>
     <property name="poolManagerStub" ref="poolmanager-stub"/>
   </bean>

skel/share/defaults/billing.properties

@@ -61,6 +61,8 @@ billing.cell.subscribe=${dcache.topic.billing}
 #
 (one-of?true|false)billing.enable.text = true
 
+(one-of?true|false)billing.format.json = false
+
 #  ---- Directory for billing logs
 #
 #   The directory within which the billing logs are to be written.
@@ -331,36 +333,42 @@ billing.text.dir = ${dcache.paths.billing}
 #           pool-to-pool transfers from door-initiated uploads or downloads.
 #
 billing.text.format.mover-info-message = $date$ [$cellType$:$cellName.cell$:$type$] [$pnfsid$,$filesize$] [$path$] $if(storage)$$$$storage.storageClass$@$storage.hsm$$$$else$<Unknown>$endif$ $transferred$ $connectionTime$ $created$ {$protocol$} [$initiator$] {$rc$:"$message$"}
+billing.json.format.mover-info-message = {"date": "$date$", "cellType": "$cellName.cell$", "type": "$type$", "pnfsid": "$pnfsid$", "filesize": "$filesize$", "path": "$path$", "storage": "$if(storage)$$$$storage.storageClass$@$storage.hsm$$$$else$<Unknown>$endif$", "transferred": "$transferred$", "connectionTime": "$connectionTime$", "created": "$created$", "protocol": "$protocol$", "initiator": "$initiator$", "returnCode": {"$rc$": "$message$"}}
 
 #  ---- RemoveFileInfoMessage
 #
 #    Submitted by PnfsManager on file removal.
 #
 billing.text.format.remove-file-info-message = $date$ [$cellType$:$cellName$:$type$] [$pnfsid$,$filesize$] [$path$] $if(storage)$$$$storage.storageClass$@$storage.hsm$$$$else$<Unknown>$endif$ {$rc$:"$message$"}
+billing.json.format.remove-file-info-message = {"date": "$date$", "cellType": "$cellType$", "cellName": "$cellName$", "type": "$type$", "pnfsid": "$pnfsid$", "filesize": "$filesize$", "path": "$path$", "storage": "$if(storage)$$$$storage.storageClass$@$storage.hsm$$$$else$<Unknown>$endif$", "returnCode": {"$rc$": "$message$"}}
 
 #  ---- DoorRequestInfoMessage
 #
 #    Submitted by doors for each file transfer.
 #
 billing.text.format.door-request-info-message = $date$ [$cellType$:$cellName$:$type$] ["$owner$":$uid$:$gid$:$clientChain$] [$pnfsid$,$filesize$] [$path$] $if(storage)$$$$storage.storageClass$@$storage.hsm$$$$else$<Unknown>$endif$ $transactionTime$ $queuingTime$ {$rc$:"$message$"}
+billing.json.format.door-request-info-message = {"date": "$date$", "cellType": "$cellType$", "cellName": "$cellName$", "type": "$type$", "owner": "$owner$", "uid": "$uid$", "gid": "$gid$", "clientChain": "$clientChain$", "pnfsid": "$pnfsid$", "filesize": "$filesize$", "path": "$path$", "storage": "$if(storage)$$$$storage.storageClass$@$storage.hsm$$$$else$<Unknown>$endif$", "transactionTime": "$transactionTime$", "queuingTime": "$queuingTime$", "returnCode": {"$rc$": "$message$"}}
 
 #  ---- StorageInfoMessage
 #
 #    Submitted by pools for each flush to and fetch from tape.
 #
 billing.text.format.storage-info-message = $date$ [$cellType$:$cellName$:$type$] [$pnfsid$,$filesize$] [$path$] $if(storage)$$$$storage.storageClass$@$storage.hsm$$$$else$<Unknown>$endif$ $transferTime$ $queuingTime$ {$rc$:"$message$"}
+billing.json.format.storage-info-message = {"date": "$date$", "cellType": "$cellType$", "cellName": "$cellName$", "type": "$type$", "path": "$path$", "storage": "$if(storage)$$$$storage.storageClass$@$storage.hsm$$$$else$<Unknown>$endif$", "transferTime": "$transferTime$", "queuingTime": "$queuingTime$", "returnCode": {"$rc$":  "$message$"}}
 
 #  ---- PoolHitInfoMessage
 #
 #    Submitted by pool manager on pool selection
 #
 billing.text.format.pool-hit-info-message = $date$ [$cellType$:$cellName.cell$:$type$] [$pnfsid$,$filesize$] [$path$] $if(storage)$$$$storage.storageClass$@$storage.hsm$$$$else$<Unknown>$endif$ $cached$ {$protocol$} {$rc$:"$message$"}
+billing.json.format.pool-hit-info-message = {"date": "$date$", "cellType": "$cellType$", "cellName": "$cellName$", "type": "$type$", "pnfsid": "$pnfsid$", "filesize": "$filesize$", "path": "$path$", "storage": "$if(storage)$$$$storage.storageClass$@$storage.hsm$$$$else$<Unknown>$endif$", "cached": "$cached$", "protocol": "$protocol$", "returnCode": {"$rc$": "$message$"}}
 
 #  ---- WarningPnfsFileInfoMessage
 #
 #    Submitted by pool manager on various failures
 #
 billing.text.format.warning-pnfs-file-info-message = $date$ [$cellType$:$cellName$:$type$] {$rc$:"$message$"}
+billing.json.format.warning-pnfs-file-info-message = {"date": "$date$", "cellType": "$cellType$", "cellName": "$cellName$", "type": "$type$", "returnCode": {"$rc$": "$message$"}}
 
 #  -----------------------------------------------------------------------
 #     Store billing data in database