From 1c304b1323eb9dd3b5cb395e8dcab82e52fc1ceb Mon Sep 17 00:00:00 2001 From: Stephan Reichhelm Date: Fri, 26 Feb 2021 10:47:22 +0100 Subject: [PATCH] * utilities for schema migration --- uncoupled/deegree-jaxb-util/pom.xml | 58 ++++ .../uncoupled/jaxbutil/AbstractScanner.java | 113 +++++++ .../jaxbutil/MetaInfSchemasMigration.java | 304 ++++++++++++++++++ .../uncoupled/jaxbutil/UpdateSchema.java | 102 ++++++ 4 files changed, 577 insertions(+) create mode 100644 uncoupled/deegree-jaxb-util/pom.xml create mode 100644 uncoupled/deegree-jaxb-util/src/main/java/org/deegree/uncoupled/jaxbutil/AbstractScanner.java create mode 100644 uncoupled/deegree-jaxb-util/src/main/java/org/deegree/uncoupled/jaxbutil/MetaInfSchemasMigration.java create mode 100644 uncoupled/deegree-jaxb-util/src/main/java/org/deegree/uncoupled/jaxbutil/UpdateSchema.java diff --git a/uncoupled/deegree-jaxb-util/pom.xml b/uncoupled/deegree-jaxb-util/pom.xml new file mode 100644 index 00000000000..5f7967759e5 --- /dev/null +++ b/uncoupled/deegree-jaxb-util/pom.xml @@ -0,0 +1,58 @@ + + 4.0.0 + org.deegree + deegree-jaxb-util + 1.0-SNAPSHOT + deegree-jaxb-util + jar + JAXB util to manipulate source tree related to deegree schema files + + + + deegree-repo + http://repo.deegree.org/content/groups/public + + never + + + true + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${java.version} + ${java.version} + ${project.build.sourceEncoding} + + + + + + + UTF-8 + ${maven.build.timestamp} + 1.8 + + + + + + org.slf4j + slf4j-simple + 1.7.30 + + + info.picocli + picocli + 4.6.1 + + + diff --git a/uncoupled/deegree-jaxb-util/src/main/java/org/deegree/uncoupled/jaxbutil/AbstractScanner.java b/uncoupled/deegree-jaxb-util/src/main/java/org/deegree/uncoupled/jaxbutil/AbstractScanner.java new file mode 100644 index 00000000000..575a3a57cb4 --- /dev/null +++ b/uncoupled/deegree-jaxb-util/src/main/java/org/deegree/uncoupled/jaxbutil/AbstractScanner.java @@ -0,0 +1,113 @@ +package org.deegree.uncoupled.jaxbutil; + +import static java.util.Collections.emptyList; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import picocli.CommandLine.Option; + +public abstract class AbstractScanner { + + private static final Logger LOG = LoggerFactory.getLogger(AbstractScanner.class); + + protected static final Pattern PAT_VERSION = Pattern.compile("^(VERSION|V|)[0-9]([._-][0-9]+)*(|-SNAPSHOT)$", + Pattern.CASE_INSENSITIVE); + + @Option(names = { "-b", "--base" }, description = "Base path (where to start scanning)") + protected String path = "../../"; + + @Option(names = { "--schemas" }, // + description = "Location where to find schemas, default to http://schemas.deegree.org") + protected String schemasUrl = "http://schemas.deegree.org"; + + public static List getFilesMatching(Path basePath, String glob) throws IOException { + if (!Files.isDirectory(basePath)) { + return emptyList(); + } + + final PathMatcher mext = FileSystems.getDefault().getPathMatcher(glob); + final List res = new LinkedList(); + + Files.walkFileTree(basePath, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + + if (!attrs.isDirectory() && mext.matches(file.getFileName())) { + LOG.trace("Lookup Soruce file {} for test", file); + res.add(file); + } + + return FileVisitResult.CONTINUE; + } + }); + + return res; + } + + public static List getFilesMatchingCompletePath(Path basePath, String glob, String... excludes) + throws IOException { + if (!Files.isDirectory(basePath)) { + return emptyList(); + } + FileSystem fs = FileSystems.getDefault(); + final PathMatcher mext = fs.getPathMatcher(glob); + List mexcludes = Arrays.stream(excludes) // + .map(fs::getPathMatcher) // + .collect(Collectors.toList()); + final List res = new LinkedList(); + + Files.walkFileTree(basePath, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + boolean exclude = false; + for (PathMatcher mexclude : mexcludes) { + if (mexclude.matches(file)) { + exclude = true; + break; + } + } + + if (!attrs.isDirectory() && !exclude && mext.matches(file)) { + LOG.trace("Lookup Soruce file {} for test", file); + res.add(file); + } + + return FileVisitResult.CONTINUE; + } + }); + + return res; + } + + protected Path removeVersion(Path p) { + Path res = null; + for (int i = 0, len = p.getNameCount(); i < len; i++) { + String seg = p.getName(i).toString(); + if (PAT_VERSION.matcher(seg).matches()) { + continue; + } + if (res == null) { + res = p.getName(i); + } else { + res = res.resolve(p.getName(i)); + } + } + return res; + } +} diff --git a/uncoupled/deegree-jaxb-util/src/main/java/org/deegree/uncoupled/jaxbutil/MetaInfSchemasMigration.java b/uncoupled/deegree-jaxb-util/src/main/java/org/deegree/uncoupled/jaxbutil/MetaInfSchemasMigration.java new file mode 100644 index 00000000000..a8048ece20f --- /dev/null +++ b/uncoupled/deegree-jaxb-util/src/main/java/org/deegree/uncoupled/jaxbutil/MetaInfSchemasMigration.java @@ -0,0 +1,304 @@ +package org.deegree.uncoupled.jaxbutil; + +import static java.util.stream.Collectors.joining; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.StreamSupport; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +@Command(name = "migrateschema", description = "Migrate Schemas to structure w/o version element") +public class MetaInfSchemasMigration extends AbstractScanner implements Runnable { + + private static final Logger LOG = LoggerFactory.getLogger(MetaInfSchemasMigration.class.getSimpleName()); + + @Option(names = { "-w", "--write" }, description = "Make changes to files") + private boolean write = false; + + public static void main(String... args) { + new CommandLine(new MetaInfSchemasMigration()).execute(args); + } + + public void run() { + try { + //moveSchemasAndExamples(); + //convertMetaInfReferences(); + convertRelative(); + //convertImport(); + } catch (Exception ex) { + LOG.error("Error {}", ex.getMessage()); + LOG.warn("Execption", ex); + } + } + + public void setWrite(boolean write) { + this.write = write; + } + + /* move files into folders without version */ + + public void moveSchemasAndExamples() throws IOException { + Path base = Paths.get(path); + LOG.info("Base Path: {}", base); + List files = getFilesMatchingCompletePath(base, "glob:**/META-INF/schemas/**", // includes + "glob:**/uncoupled/**", // excludes + "glob:**/target/**"); + LOG.info("Found {} files", files.size()); + + files.forEach(p -> { + Path relA = base.relativize(p); + try { + Path pathMove = makeMovePath(p); + if (!pathMove.equals(p)) { + Path relB = base.relativize(pathMove); + LOG.info("Moving {} -> {}", relA, relB); + if (write) { + if (!Files.isDirectory(pathMove.getParent())) { + Files.createDirectories(pathMove.getParent()); + } + Files.move(p, pathMove); + } + } + } catch (Exception ex) { + LOG.warn("[{}] failed to move, {}", relA, ex.getMessage()); + LOG.warn("Execption", ex); + } + + }); + } + + /* convert internal meta-inf references */ + + public void convertMetaInfReferences() throws IOException { + Path base = Paths.get(path); + LOG.info("Base Path: {}", base); + List files = getFilesMatchingCompletePath(base, "glob:**/*.java", // includes + "glob:**/uncoupled/**", // excludes + "glob:**/target/**"); + LOG.info("Found {} files", files.size()); + + Pattern match = Pattern.compile("\\/META-INF\\/schemas\\/[^ \"]+\\.xsd", Pattern.DOTALL); + + files.forEach(p -> { + Path rel = base.relativize(p); + try { + String content = new String(Files.readAllBytes(p)); + int lastIndex = 0; + StringBuilder output = new StringBuilder(); + + LOG.debug("- {} [size {}]", rel, content.length()); + Matcher mLoc = match.matcher(content); + while (mLoc.find()) { + output.append(content, lastIndex, mLoc.start()); + + String stringOrg = mLoc.group(); + Path pathRel = removeVersion(Paths.get(stringOrg)); + String stringRep = StreamSupport.stream(pathRel.spliterator(), false) // + .map(Path::toString) // + .collect(joining("/", "/", "")); + if (!stringOrg.equals(stringRep)) { + LOG.info("[{}] reference {} <-> {}", rel, stringOrg, stringRep); + output.append(stringRep); + } else { + output.append(stringOrg); + } + + lastIndex = mLoc.end(); + } + if (lastIndex < content.length()) { + output.append(content, lastIndex, content.length()); + } + if (write) { + Files.write(p, output.toString().getBytes()); + } + } catch (Exception ex) { + LOG.warn("{} failed to read with {}", rel, ex.getMessage()); + LOG.warn("Execption", ex); + } + }); + } + + // StringVisitor visitMeta = new + // StringVisitor("\\/META-INF\\/schemas\\/.*\\.xsd", Pattern.CASE_INSENSITIVE); + + private Path makeMovePath(Path path) throws Exception { + Path match = Paths.get("META-INF/schemas"); + for (int i = 0, len = path.getNameCount(); i < len - 1; i++) { + if (path.subpath(i, len - 1).startsWith(match)) { + Path sub = removeVersion(path.subpath(i + 2, len)); + return path.subpath(0, i + 2).resolve(sub); + } + } + throw new Exception("Could not convert path"); + } + + /* import handling */ + + public void convertRelative() throws IOException { + Path base = Paths.get(path);// .toAbsolutePath(); + LOG.info("Base Path: {}", base); + List files = getFilesMatchingCompletePath(base, // + "glob:**/META-INF/schemas/**/*.xsd", // includes + "glob:**/uncoupled/**", // excludes + "glob:**/target/**"); + LOG.info("Found {} files", files.size()); + + Pattern matchLocation = Pattern.compile("schemaLocation=\"([^\"]+)\"", Pattern.DOTALL); + Pattern matchRelative = Pattern.compile("^\\.\\.\\/[^ ]+\\.xsd$"); + files.forEach(p -> { + Path rel = base.relativize(p); + try { + Path pathFile = makeSubpathNoVersion(p); + + String content = new String(Files.readAllBytes(p)); + int lastIndex = 0; + StringBuilder output = new StringBuilder(); + + LOG.debug("- {} [size {}]", rel, content.length()); + Matcher mLoc = matchLocation.matcher(content); + while (mLoc.find()) { + output.append(content, lastIndex, mLoc.start(1)); + String contentSub = mLoc.group(1); + if ( matchRelative.matcher(contentSub).matches() ) { + Path a = pathFile.getParent(); + Path b = Paths.get(contentSub); + while ( b.getNameCount() > 1 && a.getNameCount() > 1 && "..".equals(b.getName(0).toString())) { + //LOG.info("Before {} / {}", a, b); + a = a.subpath(0, a.getNameCount() - 1); + b = b.subpath(1, b.getNameCount()); + //LOG.info("After {} / {}", a, b); + } + Path pathRef; + if (a.getNameCount() == 1 && "..".equals(b.getName(0).toString())) { + pathRef = b.subpath(1, b.getNameCount()); + } else { + pathRef = a.resolve(b); + } + String stringBase = StreamSupport.stream(pathFile.getParent().spliterator(), false) // + .map(elem -> "..") // + .collect(joining("/")); + String stringRel = StreamSupport.stream(pathRef.spliterator(), false) // + .map(Path::toString) // + .collect(joining("/")); + String newUrl = stringBase + "/" + stringRel; + if ( !contentSub.equals(newUrl)) + LOG.info( "[{}] \n {} -> {} ", rel, contentSub, newUrl ); + } else { + output.append(contentSub); + } + lastIndex = mLoc.end(1); + } + if (lastIndex < content.length()) { + output.append(content, lastIndex, content.length()); + } + + if (write) { + Files.write(p, output.toString().getBytes()); + } + } catch (Exception ex) { + LOG.warn("{} failed to read with {}", rel, ex.getMessage()); + LOG.warn("Execption", ex); + } + }); + + } + + public void convertImport() throws IOException { + Path base = Paths.get(path);// .toAbsolutePath(); + LOG.info("Base Path: {}", base); + List files = getFilesMatchingCompletePath(base, // + "glob:**/META-INF/schemas/**/*.xsd", // includes + "glob:**/uncoupled/**", // excludes + "glob:**/target/**"); + LOG.info("Found {} files", files.size()); + + Pattern matchLocation = Pattern.compile("schemaLocation=\"([^\"]+)\"", Pattern.DOTALL); + Pattern matchSchemaUrl = Pattern.compile("(?:http|https)://schemas\\.deegree\\.org[^ \"]+\\.xsd"); + + files.forEach(p -> { + Path rel = base.relativize(p); + try { + Path pathFile = makeSubpathNoVersion(p); + + String content = new String(Files.readAllBytes(p)); + int lastIndex = 0; + StringBuilder output = new StringBuilder(); + + LOG.debug("- {} [size {}]", rel, content.length()); + Matcher mLoc = matchLocation.matcher(content); + while (mLoc.find()) { + output.append(content, lastIndex, mLoc.start(1)); + + String contentSub = mLoc.group(1); + int lastIndexSub = 0; + LOG.trace("[{}] loc {}", rel, contentSub); + Matcher mUrl = matchSchemaUrl.matcher(contentSub); + while (mUrl.find()) { + output.append(contentSub, lastIndexSub, mUrl.start()); + + Path pathRef = makeSubpathNoVersion(mUrl.group()); + + String stringBase = StreamSupport.stream(pathFile.getParent().spliterator(), false) // + .map(elem -> "..") // + .collect(joining("/")); + String stringRel = StreamSupport.stream(pathRef.spliterator(), false) // + .map(Path::toString) // + .collect(joining("/")); + String newUrl = stringBase + "/" + stringRel; + + LOG.info("[{}] Change Import {} -> {}", rel, mUrl.group(), newUrl); + + output.append(newUrl); + lastIndexSub = mUrl.end(); + } + if (lastIndexSub < contentSub.length()) { + output.append(contentSub, lastIndexSub, contentSub.length()); + } + + lastIndex = mLoc.end(1); + } + if (lastIndex < content.length()) { + output.append(content, lastIndex, content.length()); + } + + if (write) { + Files.write(p, output.toString().getBytes()); + } + } catch (Exception ex) { + LOG.warn("{} failed to read with {}", rel, ex.getMessage()); + LOG.warn("Execption", ex); + } + }); + } + + private Path makeSubpathNoVersion(String uri) throws Exception { + if (uri.startsWith(schemasUrl)) { + Path p = Paths.get(uri.substring(schemasUrl.length() + 1)); + return removeVersion(p); + } + throw new Exception("Could not convert uri"); + } + + private Path makeSubpathNoVersion(Path path) throws Exception { + Path match = Paths.get("META-INF/schemas"); + for (int i = 0, len = path.getNameCount(); i < len - 1; i++) { + if (path.subpath(i, len - 1).startsWith(match)) { + String sp = path.subpath(i + 2, len).toString(); + // LOG.info("Subpath {}", sp); + return removeVersion(Paths.get(sp)); + } + } + throw new Exception("Could not convert path"); + } +} diff --git a/uncoupled/deegree-jaxb-util/src/main/java/org/deegree/uncoupled/jaxbutil/UpdateSchema.java b/uncoupled/deegree-jaxb-util/src/main/java/org/deegree/uncoupled/jaxbutil/UpdateSchema.java new file mode 100644 index 00000000000..432a93d8753 --- /dev/null +++ b/uncoupled/deegree-jaxb-util/src/main/java/org/deegree/uncoupled/jaxbutil/UpdateSchema.java @@ -0,0 +1,102 @@ +package org.deegree.uncoupled.jaxbutil; + +import static java.util.stream.Collectors.joining; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.StreamSupport; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; + +@Command(name = "updateschema", description = "Updates all found schema http(s)://schemas.deegree.org refernces") +public class UpdateSchema extends AbstractScanner implements Runnable { + + private static final Logger LOG = LoggerFactory.getLogger(UpdateSchema.class.getSimpleName()); + + @Option(names = { "-w", "--write" }, description = "Make changes to files") + private boolean write = false; + + @Parameters(index = "0", description = "Version to use in Schema references") + private String version; + + public static void main(String... args) { + new CommandLine(new UpdateSchema()).execute(args); + } + + @Override + public void run() { + try { + scanAndFix(); + } catch (Exception ex) { + LOG.error("Error {}", ex.getMessage()); + LOG.warn("Execption", ex); + } + } + + public void setWrite(boolean write) { + this.write = write; + } + + public void scanAndFix() throws IOException { + Path base = Paths.get(path);// .toAbsolutePath(); + LOG.info("Base Path: {}", base); + List files = getFilesMatchingCompletePath(base, // + "glob:**/*.{adoc,java,se,sld,xml,xsd}", // includes + "glob:**/uncoupled/**", // excludes + "glob:**/target/**"); + LOG.info("Found {} files", files.size()); + + Pattern match = Pattern.compile("((?:http|https)://schemas\\.deegree\\.org)([^ \"]+\\.xsd)"); + + files.forEach(p -> { + Path rel = base.relativize(p); + try { + String content = new String(Files.readAllBytes(p)); + int lastIndex = 0; + StringBuilder output = new StringBuilder(); + + LOG.debug("- {} [size {}]", rel, content.length()); + Matcher mLoc = match.matcher(content); + while (mLoc.find()) { + output.append(content, lastIndex, mLoc.start()); + + output.append(mLoc.group(1)); + String stringOrg = mLoc.group(2); + Path pathRel = Paths.get("/" + version).resolve(removeVersion(Paths.get(stringOrg))); + String stringRep = StreamSupport.stream(pathRel.spliterator(), false) // + .map(Path::toString) // + .collect(joining("/", "/", "")); + if (!stringOrg.equals(stringRep)) { + LOG.info("[{}] loc {} <-> {}", rel, stringOrg, stringRep); + output.append(stringRep); + } else { + output.append(stringOrg); + } + + lastIndex = mLoc.end(); + } + if (lastIndex < content.length()) { + output.append(content, lastIndex, content.length()); + } + if (write) { + Files.write(p, output.toString().getBytes()); + } + } catch (Exception ex) { + LOG.warn("{} failed to read with {}", rel, ex.getMessage()); + LOG.warn("Execption", ex); + } + }); + } + +}