diff --git a/.vscode/launch.json b/.vscode/launch.json index 4dd840e..030d9ef 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,7 +4,19 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ - + { + "type": "java", + "name": "Launch App", + "request": "launch", + "mainClass": "com.merebox.martilq.App", + "projectName": "martilq" + }, + { + "type": "java", + "name": "Launch Java Program", + "request": "launch", + "mainClass": "" + }, { "name": "PowerShell: Interactive Session", "type": "PowerShell", diff --git a/docs/samples/python/SampleGenerateBsb.py b/docs/samples/python/SampleGenerateBsb.py index 27df27e..daff6b6 100644 --- a/docs/samples/python/SampleGenerateBsb.py +++ b/docs/samples/python/SampleGenerateBsb.py @@ -29,10 +29,9 @@ def ftpPull(host, file_remote, file_local): try: ftp.login() - ftp.sendcmd('TYPE I') - with open(file_local, 'w') as fl: - res = ftp.retrlines('RETR ' + file_remote, fl.write) + with open(file_local, 'wb') as fl: + res = ftp.retrbinary(f"RETR {file_remote}", fl.write) if not res.startswith('226 Transfer complete'): print('Download failed') if os.path.isfile(file_local): @@ -59,7 +58,7 @@ for file_name in files: if file_name.endswith(".csv") | file_name.endswith(".txt"): file_remote = remote_dir + file_name file_local = "./test/" + file_name - ftpPull(remote_host, file_remote, file_local) +# ftpPull(remote_host, file_remote, file_local) print("Creating martiLQ definition") mlq = martiLQ() @@ -80,7 +79,7 @@ jsonFile.write(jd) jsonFile.close() print("Sample completed: SampleGenerateBsb.py") -lqresults, testError = mlq.TestMartiDefinition(oMarti, "./test/BSBDirectoryPlain.mti") +lqresults, testError = mlq.TestMartiDefinition("./test/BSBDirectoryPlain.mti") testfile = open("./test/LoadQualityTest01.csv", "w+", newline ="") with testfile: diff --git a/source/java/client/martilq/.editorconfig b/source/java/client/martilq/.editorconfig new file mode 100644 index 0000000..bcd1a2b --- /dev/null +++ b/source/java/client/martilq/.editorconfig @@ -0,0 +1,16 @@ +# Editor configuration, see http://editorconfig.org +root = true + +[*] +charset = utf-8 +indent_style = space +indent_size = 2 +insert_final_newline = true +trim_trailing_whitespace = true +max_line_length = 120 + +[*.sh] +end_of_line = lf + +[*.java] +indent_size = 4 diff --git a/source/java/client/martilq/.gitattributes b/source/java/client/martilq/.gitattributes new file mode 100644 index 0000000..8dfa1eb --- /dev/null +++ b/source/java/client/martilq/.gitattributes @@ -0,0 +1,2 @@ +# When shell scripts end in CRLF, bash gives a cryptic error message +*.sh text eol=lf diff --git a/source/java/client/martilq/.gitignore b/source/java/client/martilq/.gitignore new file mode 100644 index 0000000..7194620 --- /dev/null +++ b/source/java/client/martilq/.gitignore @@ -0,0 +1,28 @@ +# +# Standard Maven .gitignore +# +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties + +# +# IntelliJ +# +*.iml +.idea/* +!.idea/runConfigurations/ + +# +# Visual Studio Code +# +.settings/ +.classpath +.factorypath +.project +.vscode/ diff --git a/source/java/client/martilq/.travis.yml b/source/java/client/martilq/.travis.yml new file mode 100644 index 0000000..03dd515 --- /dev/null +++ b/source/java/client/martilq/.travis.yml @@ -0,0 +1,4 @@ +language: java +jdk: openjdk11 +after_success: + - mvn coveralls:report diff --git a/source/java/client/martilq/pom.xml b/source/java/client/martilq/pom.xml new file mode 100644 index 0000000..a68f697 --- /dev/null +++ b/source/java/client/martilq/pom.xml @@ -0,0 +1,194 @@ + + 4.0.0 + com.merebox.martilq + martilq + 0.1-SNAPSHOT + + 1.11 + 1.11 + UTF-8 + 4.12 + 3.0.0-M3 + 3.1.0 + 8.29 + 3.5.0 + 0.8.4 + 3.0.0 + 4.3.0 + + 0% + 0% + 20 + 5 + + + + + junit + junit + ${junit.version} + test + + + + + com.google.code.gson + gson + 2.8.8 + + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + ${maven-enforcer-plugin.version} + + + + enforce + + + + + 3.6.3 + + + true + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + ${maven-checkstyle-plugin.version} + + + com.puppycrawl.tools + checkstyle + ${checkstyle.version} + + + com.github.ngeor + checkstyle-rules + ${checkstyle-rules.version} + + + + com/github/ngeor/checkstyle.xml + true + ${skipTests} + + + + checkstyle + test + + check + + + + + + org.jacoco + jacoco-maven-plugin + ${jacoco-maven-plugin.version} + + + pre-unit-test + + prepare-agent + + + + post-unit-test + test + + report + + + + check-unit-test + test + + check + + + ${project.build.directory}/jacoco.exec + + + BUNDLE + + + INSTRUCTION + COVEREDRATIO + ${jacoco.unit-tests.limit.instruction-ratio} + + + BRANCH + COVEREDRATIO + ${jacoco.unit-tests.limit.branch-ratio} + + + + + CLASS + + + COMPLEXITY + TOTALCOUNT + ${jacoco.unit-tests.limit.class-complexity} + + + + + METHOD + + + COMPLEXITY + TOTALCOUNT + ${jacoco.unit-tests.limit.method-complexity} + + + + + + + + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + + + + + + travis + + + env.TRAVIS + + + + + + org.eluder.coveralls + coveralls-maven-plugin + ${coveralls-maven-plugin.version} + + + + + + diff --git a/source/java/client/martilq/src/main/java/com/merebox/Launcher.java b/source/java/client/martilq/src/main/java/com/merebox/Launcher.java new file mode 100644 index 0000000..ebc1c4f --- /dev/null +++ b/source/java/client/martilq/src/main/java/com/merebox/Launcher.java @@ -0,0 +1,38 @@ +package com.merebox; + + +import com.google.gson.*; +import com.merebox.martilq.Attribute; +import com.merebox.martilq.Resource; + +public final class Launcher { + private Launcher() { + + + } + + public static void main(String[] args) { + if (args.length > 0) + { + + MartiLQ m = new MartiLQ(); + m.title = args[0]; + + String fileName = "C:/Users/meerkat/source/marti/docs/samples/powershell/test/BSBDirectoryJul21-304.csv"; + Resource re = new Resource(fileName); + Attribute at = new Attribute(); + at.category = "cat"; + re.AddAttribute(at); + m.AddResource(re); + + Gson gson = new Gson(); + + String json = gson.toJson(m); + + System.out.println("Json " + json); + } else { + System.out.println("Hello World there! Please supply"); + + } + } +} diff --git a/source/java/client/martilq/src/main/java/com/merebox/MartiLQ.java b/source/java/client/martilq/src/main/java/com/merebox/MartiLQ.java new file mode 100644 index 0000000..ee7f367 --- /dev/null +++ b/source/java/client/martilq/src/main/java/com/merebox/MartiLQ.java @@ -0,0 +1,48 @@ +package com.merebox; + +import java.util.ArrayList; +import java.util.UUID; + +import com.merebox.martilq.*; + +public class MartiLQ { + + public String title; + public String uid = UUID.randomUUID().toString(); + public ArrayList resources = new ArrayList();; + + public String description; + public String modified; + public ArrayList tags = new ArrayList(); + public String publisher = System.getProperty("user.name"); + public String contactPoint; + public String accessLevel = "Confidential"; + public String rights = "Restricted"; + public String license; + public String state = "active"; + public Float batch = 1f; + public String describedBy; + public String landingPage; + public String theme; + + public ArrayList custom = new ArrayList(); + + public MartiLQ() + { + Object software = new Software(); + custom.add(software); + this.AddTag("document"); + this.AddTag("martiLQ"); + } + + public void AddResource(Resource resource) + { + this.resources.add(resource); + } + + public void AddTag(String tag) + { + this.tags.add(tag); + } + +} diff --git a/source/java/client/martilq/src/main/java/com/merebox/martilq/Attribute.java b/source/java/client/martilq/src/main/java/com/merebox/martilq/Attribute.java new file mode 100644 index 0000000..02c03bf --- /dev/null +++ b/source/java/client/martilq/src/main/java/com/merebox/martilq/Attribute.java @@ -0,0 +1,12 @@ +package com.merebox.martilq; + + +public class Attribute { + + public String category; + public String name; + public String function; + public String comparison; + public String value; + +} diff --git a/source/java/client/martilq/src/main/java/com/merebox/martilq/Hash.java b/source/java/client/martilq/src/main/java/com/merebox/martilq/Hash.java new file mode 100644 index 0000000..96d18c3 --- /dev/null +++ b/source/java/client/martilq/src/main/java/com/merebox/martilq/Hash.java @@ -0,0 +1,52 @@ +package com.merebox.martilq; + +import java.io.File; +import java.io.FileInputStream; +import java.security.MessageDigest; + +public class Hash { + + public String Algorithm; + public String Value; + + public static Hash GenerateHash(String algorithm, String filePath, String value) + { + + Hash hash = new Hash(); + hash.Algorithm = algorithm; + if (value == null) + { + try { + algorithm = algorithm.substring(0,3)+"-"+algorithm.substring(3); + + File file = new File(filePath); + MessageDigest digest = MessageDigest.getInstance(algorithm); + + FileInputStream fis = new FileInputStream(file); + byte[] byteArray = new byte[1024]; + int bytesCount = 0; + while ((bytesCount = fis.read(byteArray)) != -1) { + digest.update(byteArray, 0, bytesCount); + }; + fis.close(); + + byte[] bytes = digest.digest(); + StringBuilder sb = new StringBuilder(); + for(int i=0; i< bytes.length ;i++) + { + sb.append(Integer.toString((bytes[i] & 0xff) + 0x100, 16).substring(1)); + } + hash.Value =sb.toString(); + } catch (Exception ex) { + System.out.println("Exception in hash generation: "+ ex.getMessage()); + System.out.println("Algorithm: "+ algorithm); + System.out.println("File: "+ filePath); + } + } else { + hash.Value = value; + } + + return hash; + } + +} diff --git a/source/java/client/martilq/src/main/java/com/merebox/martilq/Resource.java b/source/java/client/martilq/src/main/java/com/merebox/martilq/Resource.java new file mode 100644 index 0000000..b30cef0 --- /dev/null +++ b/source/java/client/martilq/src/main/java/com/merebox/martilq/Resource.java @@ -0,0 +1,53 @@ +package com.merebox.martilq; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; + + +public class Resource { + + public String title; + public String uid = UUID.randomUUID().toString(); + public String documentName; + public Date issuedDate; + public Date modified; + public String state; + public String author = System.getProperty("user.name"); + public long length = 0; + public Hash hash; + + public String description; + public String url; + public String version; + public String format; + public String compression; + public String encryption; + + public ArrayList attributes = new ArrayList(); + + + public Resource(String filePath) + { + try { + Path path = Paths.get(filePath); + this.length = Files.size(path); + this.documentName = java.nio.file.Paths.get(filePath).getFileName().toString(); + String extPattern = "(? colCount: + colCount = len(row) + rowCount = rowCount + 1 + + self.SetAttributeValueNumber(lattribute, Category="dataset", Key="records", Function="count", Value=rowCount) + self.SetAttributeValueNumber(lattribute, Category="dataset", Key="columns", Function="count", Value=colCount) if FileType == "txt": lattribute = self.NewDefaultCsvAttributes() if ExtendedAttributes: - delimiter = "`t" rowCount = 0 colCount = 0 - # $csvData = Import-Csv $Path -Delimiter $delimiter - # foreach ($datum in $csvData) { - # $cc = (Get-Member -InputObject $datum -type NoteProperty).count - # if ($colCount -lt $cc) { - # $colCount = $cc - # } - # $rowCount += 1 - # } - self.SetAttributeValueNumber(lattribute, "records", "dataset", "count", rowCount) - self.SetAttributeValueNumber(lattribute, "columns", "dataset", "count", colCount) + + #TODO check import + import csv + + with open(PathFile, 'r') as csvfile: + datareader = csv.reader(csvfile, delimiter="\t") + for row in datareader: + if len(row) > colCount: + colCount = len(row) + rowCount = rowCount + 1 + + self.SetAttributeValueNumber(lattribute, Category="dataset", Key="records", Function="count", Value=rowCount) + self.SetAttributeValueNumber(lattribute, Category="dataset", Key="columns", Function="count", Value=colCount) if FileType == "json": @@ -521,10 +526,7 @@ class martiLQ: if FileType == "zip": lattribute = self.NewDefaultZipAttributes("ZIP") if ExtendedAttributes: - # $shell = New-Object -Com Shell.Application - # $zipFile = $shell.NameSpace($Path) - # $items = $zipFile.Items() - self.SetAttributeValueNumber(lattribute, "files", "dataset", "count", -1) + self.SetAttributeValueNumber(lattribute, "dataset", "files", "count", -1) if FileType == "7z": lattribute = self.NewDefaultZipAttributes("7Z")