billing: Add option for json format

Motivation:

See GitHub issue dCache#7421 dCache#7421

Modification:

An option to use json format was added. When it's set to true, BillingMessageSerializer is
used to generate JSON output instead of text format. Further, if json format is
set to true, there will be no header in the output file.
Also some unit tests for json format were added.

Result:

Billing files can be written in json format.

Acked-by: Tigran
Patch: https://rb.dcache.org/r/14206/
Target: master
Require-book: yes
Require-notes: yes

docs/TheBook/src/main/markdown/config-billing.md

@@ -25,6 +25,7 @@ If you installed dCache following the instructions in the Chapter [Installing dC
 ```
 
 Use the property `billing.text.dir` to set the location of the log files and the property `billing.enable.text` to control whether the plain-text log files are generated.
+To write the logs in JSON format instead of plain text, use the property `billing.format.json=true`.
 
 By default the log files are located in the directory
 `/var/lib/dcache/billing`. Under this directory the log files are

modules/dcache/src/main/java/org/dcache/services/billing/cells/BillingCell.java

@@ -40,6 +40,7 @@
 import java.util.stream.Collectors;
 import javax.annotation.PostConstruct;
 import org.dcache.cells.CellStub;
+import org.dcache.notification.BillingMessageSerializerVisitor;
 import org.dcache.services.billing.text.StringTemplateInfoMessageVisitor;
 import org.dcache.util.Args;
 import org.dcache.util.Slf4jSTErrorListener;
@@ -63,7 +64,7 @@ public final class BillingCell
 
     private static final Logger LOGGER =
           LoggerFactory.getLogger(BillingCell.class);
-    public static final String FORMAT_PREFIX = "billing.text.format.";
+    public static final String TEXT_FORMAT_PREFIX = "billing.text.format.";
 
     private final SimpleDateFormat _fileNameFormat =
           new SimpleDateFormat("yyyy.MM.dd");
@@ -87,6 +88,7 @@ public final class BillingCell
     private CellStub _poolManagerStub;
     private Path _logsDir;
     private boolean _enableText;
+    private boolean _jsonFormat;
     private boolean _flatTextDir;
 
     public BillingCell() {
@@ -102,10 +104,10 @@ public void setEnvironment(final Map<String, Object> environment) {
         };
         for (Map.Entry<String, Object> e : environment.entrySet()) {
             String key = e.getKey();
-            if (key.startsWith(FORMAT_PREFIX)) {
+            if (key.startsWith(TEXT_FORMAT_PREFIX)) {
                 String format = Formats.replaceKeywords(String.valueOf(e.getValue()), replaceable);
                 String clazz = CaseFormat.LOWER_HYPHEN.to(CaseFormat.UPPER_CAMEL,
-                      key.substring(FORMAT_PREFIX.length()));
+                      key.substring(TEXT_FORMAT_PREFIX.length()));
                 _formats.put(clazz, format);
             }
         }
@@ -195,6 +197,11 @@ public void messageArrived(Object msg) {
     }
 
     private String getFormattedMessage(InfoMessage msg) {
+        if (_jsonFormat) {
+            BillingMessageSerializerVisitor visitor = new BillingMessageSerializerVisitor();
+            msg.accept(visitor);
+            return new String(visitor.getData(), StandardCharsets.UTF_8);
+        }
         String format = _formats.get(msg.getClass().getSimpleName());
         if (!Strings.isNullOrEmpty(format)) {
             try {
@@ -352,6 +359,9 @@ private void log(Path path, String output) {
     }
 
     private String getFormatHeaders() {
+        if (_jsonFormat) {
+            return "";
+        }
         return _formats.entrySet().stream()
               .map(e -> "## " + CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_HYPHEN, e.getKey()) + ' '
                     + e.getValue() + '\n')
@@ -450,4 +460,8 @@ public void setEnableTxt(boolean enableText) {
         _enableText = enableText;
     }
 
+    public void setJsonFormat(boolean jsonFormat) {
+        _jsonFormat = jsonFormat;
+    }
+
 }

modules/dcache/src/main/resources/org/dcache/services/billing/cells/billing.xml

@@ -39,6 +39,7 @@
     class="org.dcache.services.billing.cells.BillingCell">
     <property name="logsDir" value="${billing.text.dir}"/>
     <property name="enableTxt" value="${billing.enable.text}"/>
+    <property name="jsonFormat" value="${billing.format.json}"/>
     <property name="flatTextDir" value="${billing.text.flat-dir}"/>
     <property name="poolManagerStub" ref="poolmanager-stub"/>
   </bean>

modules/dcache/src/test/java/org/dcache/services/billing/text/BillingJsonTest.java

@@ -0,0 +1,115 @@
+package org.dcache.services.billing.text;
+
+import static org.junit.Assert.assertTrue;
+
+import diskCacheV111.util.PnfsId;
+import diskCacheV111.vehicles.DoorRequestInfoMessage;
+import diskCacheV111.vehicles.GenericStorageInfo;
+import diskCacheV111.vehicles.MoverInfoMessage;
+import diskCacheV111.vehicles.PoolHitInfoMessage;
+import diskCacheV111.vehicles.RemoveFileInfoMessage;
+import diskCacheV111.vehicles.StorageInfoMessage;
+import diskCacheV111.vehicles.WarningPnfsFileInfoMessage;
+import dmg.cells.nucleus.CellAddressCore;
+import java.nio.charset.StandardCharsets;
+import javax.security.auth.Subject;
+import org.dcache.mock.ProtocolInfoBuilder;
+import org.dcache.notification.BillingMessageSerializerVisitor;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class BillingJsonTest {
+
+    private static final CellAddressCore cellAdressCore = new CellAddressCore(
+          "bccs_uib_no_023@nas023_bccs_uib_no_1Domain");
+    private static final PnfsId pnfsid = new PnfsId("0000B706DD4045F346F2B90F882B706DA807");
+    private static final BillingMessageSerializerVisitor visitor = new BillingMessageSerializerVisitor();
+    GenericStorageInfo si = new GenericStorageInfo();
+
+    @Before
+    public void setup() {
+        si.setHsm("osm");
+        si.setStorageClass("atlas:default");
+    }
+
+    @Test
+    public void testDoorRequestInfoMessageValidJson() {
+        DoorRequestInfoMessage msg = new DoorRequestInfoMessage(cellAdressCore);
+        msg.setSubject(new Subject());
+
+        msg.accept(visitor);
+
+        assertTrue(isValidJson(new String(visitor.getData(), StandardCharsets.UTF_8)));
+    }
+
+    @Test
+    public void testMoverInfoMessageValidJson() {
+        MoverInfoMessage msg = new MoverInfoMessage(cellAdressCore, pnfsid);
+        msg.setFileCreated(true);
+        msg.setFileSize(687926);
+        msg.setTransaction("remove");
+        msg.setSubject(new Subject());
+        msg.setStorageInfo(si);
+        msg.setTransferAttributes(256437, 2784,
+              ProtocolInfoBuilder.aProtocolInfo().withProtocol("GFtp")
+                    .withIPAddress("109.105.124.147").build());
+
+        msg.accept(visitor);
+
+        assertTrue(isValidJson(new String(visitor.getData(), StandardCharsets.UTF_8)));
+    }
+
+    @Test
+    public void testPoolHitInfoMessageValidJson() {
+        PoolHitInfoMessage msg = new PoolHitInfoMessage(cellAdressCore, pnfsid);
+
+        msg.accept(visitor);
+
+        assertTrue(isValidJson(new String(visitor.getData(), StandardCharsets.UTF_8)));
+    }
+
+    @Test
+    public void testRemoveFileInfoMessageValidJson() {
+        RemoveFileInfoMessage msg = new RemoveFileInfoMessage(cellAdressCore, pnfsid);
+        msg.setSubject(new Subject());
+
+        msg.setStorageInfo(si);
+
+        msg.accept(visitor);
+
+        assertTrue(isValidJson(new String(visitor.getData(), StandardCharsets.UTF_8)));
+    }
+
+    @Test
+    public void testStorageInfoMessageValidJson() {
+        StorageInfoMessage msg = new StorageInfoMessage(cellAdressCore, pnfsid, true);
+        msg.setStorageInfo(si);
+
+        msg.accept(visitor);
+
+        assertTrue(isValidJson(new String(visitor.getData(), StandardCharsets.UTF_8)));
+    }
+
+    @Test
+    public void testWarningPnfsFileInfoMessageValidJson() {
+        WarningPnfsFileInfoMessage msg = new WarningPnfsFileInfoMessage("TODO", cellAdressCore,
+              pnfsid, 0, "TODO");
+
+        msg.accept(visitor);
+
+        assertTrue(isValidJson(new String(visitor.getData(), StandardCharsets.UTF_8)));
+    }
+
+    private boolean isValidJson(String string) {
+        try {
+            new JSONObject(string);
+            return true;
+        } catch (JSONException e) {
+            return false;
+        }
+    }
+
+}

skel/share/defaults/billing.properties

@@ -61,6 +61,12 @@ billing.cell.subscribe=${dcache.topic.billing}
 #
 (one-of?true|false)billing.enable.text = true
 
+#  ---- Use JSON format
+#
+#   Changes the plain text output to JSON format if set to 'true'.
+#
+(one-of?true|false)billing.format.json = false
+
 #  ---- Directory for billing logs
 #
 #   The directory within which the billing logs are to be written.