Including Java

Samples corrected

.vscode/launch.json

@@ -4,7 +4,19 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
-        
+        {
+            "type": "java",
+            "name": "Launch App",
+            "request": "launch",
+            "mainClass": "com.merebox.martilq.App",
+            "projectName": "martilq"
+        },
+        {
+            "type": "java",
+            "name": "Launch Java Program",
+            "request": "launch",
+            "mainClass": ""
+        },
         {
             "name": "PowerShell: Interactive Session",
             "type": "PowerShell",

docs/samples/python/SampleGenerateBsb.py

@@ -29,10 +29,9 @@ def ftpPull(host, file_remote, file_local):
                 
         try:
             ftp.login()  
-            ftp.sendcmd('TYPE I') 
             
-            with open(file_local, 'w') as fl:
+            with open(file_local, 'wb') as fl:
-                res = ftp.retrlines('RETR ' + file_remote, fl.write)
+                res = ftp.retrbinary(f"RETR {file_remote}", fl.write)
                 if not res.startswith('226 Transfer complete'):
                     print('Download failed')
                     if os.path.isfile(file_local):
@@ -59,7 +58,7 @@ for file_name in files:
         if file_name.endswith(".csv") | file_name.endswith(".txt"):
             file_remote = remote_dir + file_name
             file_local = "./test/" + file_name
-            ftpPull(remote_host, file_remote, file_local)
+#            ftpPull(remote_host, file_remote, file_local)
 
 print("Creating martiLQ definition")
 mlq = martiLQ()
@@ -80,7 +79,7 @@ jsonFile.write(jd)
 jsonFile.close()
 print("Sample completed: SampleGenerateBsb.py")
 
-lqresults, testError = mlq.TestMartiDefinition(oMarti, "./test/BSBDirectoryPlain.mti")
+lqresults, testError = mlq.TestMartiDefinition("./test/BSBDirectoryPlain.mti")
 
 testfile = open("./test/LoadQualityTest01.csv", "w+", newline ="") 
 with testfile:     

source/java/client/martilq/.editorconfig

@@ -0,0 +1,16 @@
+# Editor configuration, see http://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+insert_final_newline = true
+trim_trailing_whitespace = true
+max_line_length = 120
+
+[*.sh]
+end_of_line = lf
+
+[*.java]
+indent_size = 4

source/java/client/martilq/.gitattributes

@@ -0,0 +1,2 @@
+# When shell scripts end in CRLF, bash gives a cryptic error message
+*.sh text eol=lf

source/java/client/martilq/.gitignore

@@ -0,0 +1,28 @@
+#
+# Standard Maven .gitignore
+#
+target/
+pom.xml.tag
+pom.xml.releaseBackup
+pom.xml.versionsBackup
+pom.xml.next
+release.properties
+dependency-reduced-pom.xml
+buildNumber.properties
+.mvn/timing.properties
+
+#
+# IntelliJ
+#
+*.iml
+.idea/*
+!.idea/runConfigurations/
+
+#
+# Visual Studio Code
+#
+.settings/
+.classpath
+.factorypath
+.project
+.vscode/

source/java/client/martilq/.travis.yml

@@ -0,0 +1,4 @@
+language: java
+jdk: openjdk11
+after_success:
+  - mvn coveralls:report

source/java/client/martilq/pom.xml

@@ -0,0 +1,194 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.merebox.martilq</groupId>
+  <artifactId>martilq</artifactId>
+  <version>0.1-SNAPSHOT</version>
+  <properties>
+    <maven.compiler.source>1.11</maven.compiler.source>
+    <maven.compiler.target>1.11</maven.compiler.target>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <junit.version>4.12</junit.version>
+    <maven-enforcer-plugin.version>3.0.0-M3</maven-enforcer-plugin.version>
+    <maven-checkstyle-plugin.version>3.1.0</maven-checkstyle-plugin.version>
+    <checkstyle.version>8.29</checkstyle.version>
+    <checkstyle-rules.version>3.5.0</checkstyle-rules.version>
+    <jacoco-maven-plugin.version>0.8.4</jacoco-maven-plugin.version>
+    <maven-javadoc-plugin.version>3.0.0</maven-javadoc-plugin.version>
+    <coveralls-maven-plugin.version>4.3.0</coveralls-maven-plugin.version>
+    <!-- JaCoCo thresholds. Increase gradually as you add tests. -->
+    <jacoco.unit-tests.limit.instruction-ratio>0%</jacoco.unit-tests.limit.instruction-ratio>
+    <jacoco.unit-tests.limit.branch-ratio>0%</jacoco.unit-tests.limit.branch-ratio>
+    <jacoco.unit-tests.limit.class-complexity>20</jacoco.unit-tests.limit.class-complexity>
+    <jacoco.unit-tests.limit.method-complexity>5</jacoco.unit-tests.limit.method-complexity>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>${junit.version}</version>
+      <scope>test</scope>
+    </dependency>
+
+<!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->
+<dependency>
+    <groupId>com.google.code.gson</groupId>
+    <artifactId>gson</artifactId>
+    <version>2.8.8</version>
+</dependency>
+
+
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-enforcer-plugin</artifactId>
+        <version>${maven-enforcer-plugin.version}</version>
+        <executions>
+          <execution>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <configuration>
+              <rules>
+                <requireMavenVersion>
+                  <version>3.6.3</version>
+                </requireMavenVersion>
+              </rules>
+              <fail>true</fail>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-checkstyle-plugin</artifactId>
+        <version>${maven-checkstyle-plugin.version}</version>
+        <dependencies>
+          <dependency>
+            <groupId>com.puppycrawl.tools</groupId>
+            <artifactId>checkstyle</artifactId>
+            <version>${checkstyle.version}</version>
+          </dependency>
+          <dependency>
+            <groupId>com.github.ngeor</groupId>
+            <artifactId>checkstyle-rules</artifactId>
+            <version>${checkstyle-rules.version}</version>
+          </dependency>
+        </dependencies>
+        <configuration>
+          <configLocation>com/github/ngeor/checkstyle.xml</configLocation>
+          <includeTestSourceDirectory>true</includeTestSourceDirectory>
+          <skip>${skipTests}</skip>
+        </configuration>
+        <executions>
+          <execution>
+            <id>checkstyle</id>
+            <phase>test</phase>
+            <goals>
+              <goal>check</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.jacoco</groupId>
+        <artifactId>jacoco-maven-plugin</artifactId>
+        <version>${jacoco-maven-plugin.version}</version>
+        <executions>
+          <execution>
+            <id>pre-unit-test</id>
+            <goals>
+              <goal>prepare-agent</goal>
+            </goals>
+          </execution>
+          <execution>
+            <id>post-unit-test</id>
+            <phase>test</phase>
+            <goals>
+              <goal>report</goal>
+            </goals>
+          </execution>
+          <execution>
+            <id>check-unit-test</id>
+            <phase>test</phase>
+            <goals>
+              <goal>check</goal>
+            </goals>
+            <configuration>
+              <dataFile>${project.build.directory}/jacoco.exec</dataFile>
+              <rules>
+                <rule>
+                  <element>BUNDLE</element>
+                  <limits>
+                    <limit>
+                      <counter>INSTRUCTION</counter>
+                      <value>COVEREDRATIO</value>
+                      <minimum>${jacoco.unit-tests.limit.instruction-ratio}</minimum>
+                    </limit>
+                    <limit>
+                      <counter>BRANCH</counter>
+                      <value>COVEREDRATIO</value>
+                      <minimum>${jacoco.unit-tests.limit.branch-ratio}</minimum>
+                    </limit>
+                  </limits>
+                </rule>
+                <rule>
+                  <element>CLASS</element>
+                  <limits>
+                    <limit>
+                      <counter>COMPLEXITY</counter>
+                      <value>TOTALCOUNT</value>
+                      <maximum>${jacoco.unit-tests.limit.class-complexity}</maximum>
+                    </limit>
+                  </limits>
+                </rule>
+                <rule>
+                  <element>METHOD</element>
+                  <limits>
+                    <limit>
+                      <counter>COMPLEXITY</counter>
+                      <value>TOTALCOUNT</value>
+                      <maximum>${jacoco.unit-tests.limit.method-complexity}</maximum>
+                    </limit>
+                  </limits>
+                </rule>
+              </rules>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+  <reporting>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <version>${maven-javadoc-plugin.version}</version>
+      </plugin>
+    </plugins>
+  </reporting>
+  <profiles>
+    <!-- Publish coverage report to Coveralls, only when running in Travis. -->
+    <profile>
+      <id>travis</id>
+      <activation>
+        <property>
+          <name>env.TRAVIS</name>
+        </property>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.eluder.coveralls</groupId>
+            <artifactId>coveralls-maven-plugin</artifactId>
+            <version>${coveralls-maven-plugin.version}</version>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
+</project>

source/java/client/martilq/src/main/java/com/merebox/Launcher.java

@@ -0,0 +1,38 @@
+package com.merebox;
+
+
+import com.google.gson.*;
+import com.merebox.martilq.Attribute;
+import com.merebox.martilq.Resource;
+
+public final class Launcher {
+    private Launcher() {
+
+
+    }
+
+    public static void main(String[] args) {
+        if (args.length > 0)
+        {
+
+            MartiLQ m = new MartiLQ();
+            m.title = args[0];
+        
+            String fileName = "C:/Users/meerkat/source/marti/docs/samples/powershell/test/BSBDirectoryJul21-304.csv";
+            Resource re = new Resource(fileName);
+            Attribute at = new Attribute();
+            at.category = "cat";
+            re.AddAttribute(at);
+            m.AddResource(re);
+
+            Gson gson = new Gson();
+
+            String json = gson.toJson(m);
+
+            System.out.println("Json " + json);
+        }  else {
+            System.out.println("Hello World there! Please supply");
+
+        }
+    }
+}

source/java/client/martilq/src/main/java/com/merebox/MartiLQ.java

@@ -0,0 +1,48 @@
+package com.merebox;
+
+import java.util.ArrayList;
+import java.util.UUID;
+
+import com.merebox.martilq.*;
+
+public class MartiLQ {
+
+    public String title;
+    public String uid = UUID.randomUUID().toString();
+    public ArrayList<Resource> resources = new ArrayList<Resource>();;
+
+    public String description;
+    public String modified;
+    public ArrayList<String> tags = new ArrayList<String>();
+    public String publisher = System.getProperty("user.name");
+    public String contactPoint;
+    public String accessLevel = "Confidential";
+    public String rights = "Restricted";
+    public String license;
+    public String state = "active";
+    public Float batch = 1f;
+    public String describedBy;
+    public String landingPage;
+    public String theme;
+
+    public ArrayList<Object> custom = new ArrayList<Object>();
+
+    public MartiLQ()
+    {
+        Object software = new Software();
+        custom.add(software);
+        this.AddTag("document");
+        this.AddTag("martiLQ");
+    }
+
+    public void AddResource(Resource resource)
+    {
+        this.resources.add(resource);
+    }
+
+    public void AddTag(String tag)
+    {
+        this.tags.add(tag);
+    }
+
+}

source/java/client/martilq/src/main/java/com/merebox/martilq/Attribute.java

@@ -0,0 +1,12 @@
+package com.merebox.martilq;
+
+
+public class Attribute {
+
+    public String category;
+    public String name;
+    public String function;
+    public String comparison;
+    public String value;
+
+}

source/java/client/martilq/src/main/java/com/merebox/martilq/Hash.java

@@ -0,0 +1,52 @@
+package com.merebox.martilq;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.security.MessageDigest;
+
+public class Hash {
+    
+    public String Algorithm;
+    public String Value;
+    
+    public static Hash GenerateHash(String algorithm, String filePath, String value)
+    {
+
+        Hash hash = new Hash();
+        hash.Algorithm = algorithm;
+        if (value == null)
+        {
+            try {
+                algorithm = algorithm.substring(0,3)+"-"+algorithm.substring(3);
+
+                File file = new File(filePath);
+                MessageDigest digest = MessageDigest.getInstance(algorithm);
+    
+                FileInputStream fis = new FileInputStream(file);
+                byte[] byteArray = new byte[1024];
+                int bytesCount = 0; 
+                while ((bytesCount = fis.read(byteArray)) != -1) {
+                    digest.update(byteArray, 0, bytesCount);
+                };
+                fis.close();
+                
+                byte[] bytes = digest.digest();
+                StringBuilder sb = new StringBuilder();
+                for(int i=0; i< bytes.length ;i++)
+                {
+                    sb.append(Integer.toString((bytes[i] & 0xff) + 0x100, 16).substring(1));
+                }
+                hash.Value =sb.toString();
+            } catch (Exception ex) {
+                System.out.println("Exception in hash generation: "+ ex.getMessage());
+                System.out.println("Algorithm: "+ algorithm);
+                System.out.println("File: "+ filePath);
+            }
+        } else {
+            hash.Value = value;
+        }
+
+        return hash;
+    }
+
+}

source/java/client/martilq/src/main/java/com/merebox/martilq/Resource.java

@@ -0,0 +1,53 @@
+package com.merebox.martilq;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+
+
+public class Resource {
+
+    public String title;
+    public String uid = UUID.randomUUID().toString();
+    public String documentName;
+    public Date issuedDate;
+    public Date modified;
+    public String state;
+    public String author = System.getProperty("user.name");
+    public long length = 0;
+    public Hash hash;
+
+    public String description;
+    public String url;
+    public String version;
+    public String format;
+    public String compression;
+    public String encryption;
+
+    public ArrayList<Attribute> attributes = new ArrayList<Attribute>();
+
+    
+    public Resource(String filePath)
+    {
+        try {
+            Path path = Paths.get(filePath);
+            this.length = Files.size(path);
+            this.documentName = java.nio.file.Paths.get(filePath).getFileName().toString();
+            String extPattern = "(?<!^)[.][^.]*$";
+            this.title = this.documentName.replaceAll(extPattern, "");
+            this.hash = Hash.GenerateHash("SHA512", filePath, null);
+        } catch (IOException ex) {
+            System.out.println("Exception in resource generation: "+ ex.getMessage());
+            System.out.println("File: "+ filePath);
+        }
+    }
+
+    public void AddAttribute(Attribute attribute)
+    {
+        this.attributes.add(attribute);
+    }
+
+
+}

source/java/client/martilq/src/main/java/com/merebox/martilq/Software.java

@@ -0,0 +1,11 @@
+package com.merebox.martilq;
+
+
+public class Software {
+
+    public String extension = "software";
+    public String softwareName = "MARTILQREFERENCE";
+    public String author = "Meerkat@merebox.com";
+    public String version = "0.0.1";
+
+}

source/java/client/martilq/src/test/java/com/merebox/martilq/LauncherTest.java

@@ -0,0 +1,18 @@
+package com.merebox.martilq;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * Unit test for simple App.
+ */
+public class LauncherTest {
+    /**
+     * Rigorous Test.
+     */
+    @Test
+    public void testApp() {
+        assertTrue(true);
+    }
+}

source/powershell/MartiLQ.ps1

@@ -94,7 +94,7 @@ function New-MartiDefinition
         rights = "Restricted"
         license = ""
         state = "active"
-        batch = 1
+        batch = 1.0
         describedBy = ""
         landingPage = ""
         theme =""

source/python/client/martiLQ.py

@@ -126,7 +126,7 @@ class martiLQ:
             "rights": "Restricted",
             "license": "",
             "state": "active",
-            "batch": 1,
+            "batch": 1.0,
             "describedBy": "",
             "landingPage": "",
             "theme": "", 
@@ -168,7 +168,7 @@ class martiLQ:
             if ExcludeHash:
                 hash = None
             else:
-                hash = self.NewMartiHash(Algorithm=HashAlgorithm, FilePath=SourcePath, Value="")
+                hash = self.NewMartiLQHash(Algorithm=HashAlgorithm, FilePath=SourcePath, Value="")
 
             lattribute = self.SetMartiResourceAttributes(SourcePath, os.path.splitext(SourcePath)[1][1:], ExtendAttributes)
 
@@ -211,7 +211,7 @@ class martiLQ:
 
 
 
-    def NewMartiHash(self, Algorithm, FilePath, Value=""):
+    def NewMartiLQHash(self, Algorithm, FilePath, Value=""):
         
         if Value  == "" and FilePath != "":
             if Algorithm == "SHA256":
@@ -450,7 +450,7 @@ class martiLQ:
     def SetAttributeValueNumber(self, Attributes,Category,Key,Function,Value,Comparison = "EQ"):
 
         for item in Attributes:
-        
+
             if item["category"] == Category and item["name"] == Key and item["function"] == Function:
                 if item["comparison"] == "NA" or item["comparison"] == Comparison:
                     item["comparison"] = Comparison
@@ -472,7 +472,7 @@ class martiLQ:
         return
 
 
-    def SetMartiResourceAttributes(self, Path,FileType,ExtendedAttributes):
+    def SetMartiResourceAttributes(self, PathFile, FileType, ExtendedAttributes):
 
         lattribute = None
 
@@ -483,35 +483,40 @@ class martiLQ:
                 delimiter = ","
                 rowCount = 0
                 colCount = 0
-                # csvData = Import-Csv $Path -Delimiter $delimiter 
+
-                # foreach ($datum in $csvData) {
+                #TODO check import
-                #     $cc = (Get-Member -InputObject $datum -type NoteProperty).count
+                import csv
-                #     if ($colCount -lt $cc) {
+
-                #         $colCount = $cc
+                with open(PathFile, 'r') as csvfile:
-                #     }
+                    datareader = csv.reader(csvfile, delimiter=",")
-                #     $rowCount += 1
+                    for row in datareader:
-                # }
+                        if len(row) > colCount:
-                self.SetAttributeValueNumber(lattribute, "records", "dataset", "count", rowCount)
+                            colCount = len(row)
-                self.SetAttributeValueNumber(lattribute, "columns", "dataset", "count", colCount)
+                        rowCount = rowCount + 1
+
+                self.SetAttributeValueNumber(lattribute, Category="dataset", Key="records", Function="count", Value=rowCount)
+                self.SetAttributeValueNumber(lattribute, Category="dataset", Key="columns", Function="count", Value=colCount)
 
 
         if FileType == "txt":
             lattribute = self.NewDefaultCsvAttributes()
 
             if ExtendedAttributes:
-                delimiter = "`t"
                 rowCount = 0
                 colCount = 0
-                # $csvData = Import-Csv $Path -Delimiter $delimiter 
+
-                # foreach ($datum in $csvData) {
+                #TODO check import
-                #     $cc = (Get-Member -InputObject $datum -type NoteProperty).count
+                import csv
-                #     if ($colCount -lt $cc) {
+
-                #         $colCount = $cc
+                with open(PathFile, 'r') as csvfile:
-                #     }
+                    datareader = csv.reader(csvfile, delimiter="\t")
-                #     $rowCount += 1
+                    for row in datareader:
-                # }
+                        if len(row) > colCount:
-                self.SetAttributeValueNumber(lattribute, "records", "dataset", "count", rowCount)
+                            colCount = len(row)
-                self.SetAttributeValueNumber(lattribute, "columns", "dataset", "count", colCount)
+                        rowCount = rowCount + 1
+
+                self.SetAttributeValueNumber(lattribute, Category="dataset", Key="records", Function="count", Value=rowCount)
+                self.SetAttributeValueNumber(lattribute, Category="dataset", Key="columns", Function="count", Value=colCount)
 
 
         if FileType == "json":
@@ -521,10 +526,7 @@ class martiLQ:
         if FileType == "zip":
             lattribute = self.NewDefaultZipAttributes("ZIP")
             if ExtendedAttributes:
-                # $shell = New-Object -Com Shell.Application
+                self.SetAttributeValueNumber(lattribute, "dataset", "files", "count", -1)
-                # $zipFile = $shell.NameSpace($Path)
-                # $items = $zipFile.Items()
-                self.SetAttributeValueNumber(lattribute, "files", "dataset", "count", -1)
 
         if FileType == "7z":
             lattribute = self.NewDefaultZipAttributes("7Z")