diff --git a/README.md b/README.md index 46b3a86..c25ec77 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ This repo is a open source tool to check a connection status of URLs found from - Redirecting to followed location when it is 301, 307 and 308 - Checking a version of archived URLs using the WayBackMachine - Checking whether Urls work with https or not + - Allow to pass directory paths and recursively process all children under that directory ## Getting Started diff --git a/src/UrlCheck.java b/src/UrlCheck.java index f570777..86cec8b 100644 --- a/src/UrlCheck.java +++ b/src/UrlCheck.java @@ -1,5 +1,6 @@ import java.io.BufferedReader; +import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; @@ -7,100 +8,123 @@ import java.net.URL; import java.net.URLConnection; import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.HashSet; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.FileVisitResult; +import java.nio.file.attribute.BasicFileAttributes; + public class UrlCheck { - // url set regex - final static String regex = "(https?):\\/\\/[-a-zA-Z0-9+&@#%?=~_|!:,.;]*[-a-zA-Z0-9+&@#%=~_|\\/]*"; - - // delimiter to get url from input file - final static String delimiter = "[\\[\\]\"<>'\n\b\r]"; - - - public static void helpMessage() - { - System.out.println("========================================================"); - System.out.println("| Help message |"); - System.out.println("========================================================"); - System.out.println("| Please Type proper argument |"); - System.out.println("| |"); - System.out.println("| 1) UrlCheck |"); - System.out.println("| ex) UrlCheck index.html |"); - System.out.println("| UrlCheck index.html index2.html |"); - System.out.println("| |"); - System.out.println("| 2) UrlCheck help |"); - System.out.println("| |"); - System.out.println("| 3) Option a, s, v, m |"); - System.out.println("| |"); - System.out.println("| For Window : |"); - System.out.println("| |"); - System.out.println("| Option a : To check for archived versions of URLs |"); - System.out.println("| |"); - System.out.println("| UrlCheck --a |"); - System.out.println("| ex) UrlCheck --a index2.html |"); - System.out.println("| |"); - System.out.println("| Option s : To request URLs with https |"); - System.out.println("| |"); - System.out.println("| UrlCheck --s |"); - System.out.println("| ex) UrlCheck --s index2.html |"); - System.out.println("| |"); - System.out.println("| Option v : To check the version |"); - System.out.println("| |"); - System.out.println("| UrlCheck --v |"); - System.out.println("| UrlCheck --version |"); - System.out.println("| |"); - System.out.println("| |"); - System.out.println("| For Mac : |"); - System.out.println("| |"); - System.out.println("| Option m : To run the tool in Mac |"); - System.out.println("| |"); - System.out.println("| UrlCheck /m |"); - System.out.println("| ex) UrlCheck /m index2.html |"); - System.out.println("| |"); - System.out.println("| Option a : To check for archived versions of URLs |"); - System.out.println("| |"); - System.out.println("| UrlCheck /a |"); - System.out.println("| ex) UrlCheck /a index2.html |"); - System.out.println("| |"); - System.out.println("| Option s : To request URLs with https |"); - System.out.println("| |"); - System.out.println("| UrlCheck /s |"); - System.out.println("| ex) UrlCheck /s index2.html |"); - System.out.println("| |"); - System.out.println("| Option v : To check the version |"); - System.out.println("| |"); - System.out.println("| UrlCheck /v |"); - System.out.println("| |"); - System.out.println("| Thanks! |"); - - } - - public static void startMessage() - { - System.out.println("========================================================"); - System.out.println("| URL TEST RUNNING |"); - System.out.println("========================================================"); + // url set regex + final static String regex = "(https?):\\/\\/[-a-zA-Z0-9+&@#%?=~_|!:,.;]*[-a-zA-Z0-9+&@#%=~_|\\/]*"; - } - - public static void endMessage() - { - - System.out.println("========================================================"); - } - + // delimiter to get url from input file + final static String delimiter = "[\\[\\]\"<>'\n\b\r]"; + + + public static void helpMessage() + { + System.out.println("========================================================"); + System.out.println("| Help message |"); + System.out.println("========================================================"); + System.out.println("| Please Type proper argument |"); + System.out.println("| |"); + System.out.println("| 1) UrlCheck |"); + System.out.println("| ex) UrlCheck index.html |"); + System.out.println("| UrlCheck index.html index2.html |"); + System.out.println("| |"); + System.out.println("| 2) UrlCheck help |"); + System.out.println("| |"); + System.out.println("| 3) Option a, s, v, m, d |"); + System.out.println("| |"); + System.out.println("| For Window : |"); + System.out.println("| |"); + System.out.println("| Option a : To check for archived versions of URLs |"); + System.out.println("| |"); + System.out.println("| UrlCheck --a |"); + System.out.println("| ex) UrlCheck --a index2.html |"); + System.out.println("| |"); + System.out.println("| Option s : To request URLs with https |"); + System.out.println("| |"); + System.out.println("| UrlCheck --s |"); + System.out.println("| ex) UrlCheck --s index2.html |"); + System.out.println("| |"); + System.out.println("| Option v : To check the version |"); + System.out.println("| |"); + System.out.println("| UrlCheck --v |"); + System.out.println("| UrlCheck --version |"); + System.out.println("| |"); + System.out.println("| |"); + System.out.println("| For Mac : |"); + System.out.println("| |"); + System.out.println("| Option m : To run the tool in Mac |"); + System.out.println("| |"); + System.out.println("| UrlCheck /m |"); + System.out.println("| ex) UrlCheck /m index2.html |"); + System.out.println("| |"); + System.out.println("| Option a : To check for archived versions of URLs |"); + System.out.println("| |"); + System.out.println("| UrlCheck /a |"); + System.out.println("| ex) UrlCheck /a index2.html |"); + System.out.println("| |"); + System.out.println("| Option s : To request URLs with https |"); + System.out.println("| |"); + System.out.println("| UrlCheck /s |"); + System.out.println("| ex) UrlCheck /s index2.html |"); + System.out.println("| |"); + System.out.println("| Option v : To check the version |"); + System.out.println("| |"); + System.out.println("| UrlCheck /v |"); + System.out.println("| |"); + System.out.println("| Thanks! |"); + + } + + public static void startMessage() + { + System.out.println("========================================================"); + System.out.println("| URL TEST RUNNING |"); + System.out.println("========================================================"); + + } + + public static void endMessage() + { + + System.out.println("========================================================"); + } - // list up url in input file - public static void fileUrlListUp(String fName, boolean archived, boolean secured, boolean runMac) - { - String regSecure = "^(http)://"; - Pattern pat = Pattern.compile(regex, Pattern.MULTILINE); - - BufferedReader br; - try { - br = new BufferedReader(new FileReader(fName)); + // list up url in input file + public static void fileUrlListUp(String fName, boolean archived, boolean secured, boolean runMac) + { + String regSecure = "^(http)://"; + Pattern pat = Pattern.compile(regex, Pattern.MULTILINE); + + try { + File file=new File(fName); + ArrayList files =new ArrayList(); + + if(file.isDirectory()) { + files = visitFileRecursive(fName); + } + else { + files.add(fName); + } + + for(String f: files) { + + System.out.println("\n\"" + f + "\" is checking..."); + + BufferedReader br; + + br = new BufferedReader(new FileReader(f)); String line = null; while((line = br.readLine()) != null) { @@ -112,7 +136,7 @@ public static void fileUrlListUp(String fName, boolean archived, boolean secured { // change http to https if(secured) str = str.replaceFirst(regSecure, "https://"); - + if(runMac) { UrlCheckForMac.availableURL(str); @@ -121,60 +145,61 @@ public static void fileUrlListUp(String fName, boolean archived, boolean secured { UrlCheckForWindow.availableURL(str); } - + // request archived if(archived) archiveUrl(str); } } } br.close(); - } catch (FileNotFoundException e) { - - System.out.println("File not Found"); - } catch (IOException e) { - System.out.println("File not Found"); } + } catch (FileNotFoundException e) { + System.out.println("File not Found"); + } catch (IOException e) { + System.out.println("File not Found"); } - - // Archive API - public static void archiveUrl(String host) - { - - String apiUrl = "http://archive.org/wayback/available?url=" + host; - StringBuilder sb = new StringBuilder(); - - - try { - URL url = new URL(apiUrl); - URLConnection con; - con = url.openConnection(); - - - BufferedReader brd = new BufferedReader(new InputStreamReader(con.getInputStream(),Charset.defaultCharset())); - if(brd != null) + + } + + // Archive API + public static void archiveUrl(String host) + { + + String apiUrl = "http://archive.org/wayback/available?url=" + host; + StringBuilder sb = new StringBuilder(); + + + try { + URL url = new URL(apiUrl); + URLConnection con; + con = url.openConnection(); + + + BufferedReader brd = new BufferedReader(new InputStreamReader(con.getInputStream(),Charset.defaultCharset())); + if(brd != null) + { + int cp; + while((cp = brd.read())!= -1) { - int cp; - while((cp = brd.read())!= -1) - { - sb.append((char)cp); - } - brd.close(); + sb.append((char)cp); } - System.out.println(sb); - - }catch(Exception e) - { - throw new RuntimeException("Exception URL : "+ host, e); + brd.close(); } + System.out.println(sb); + + }catch(Exception e) + { + throw new RuntimeException("Exception URL : "+ host, e); } - + } + public static void main(String[] args) { - + boolean archived = false; boolean secured = false; boolean runMac = false; - + if(args.length == 0 || args[0].toLowerCase().equals("help")) { helpMessage(); @@ -182,8 +207,8 @@ public static void main(String[] args) { else { startMessage(); - - // command line flag a, s + + // command line flag a, s, d if(args[0].startsWith("--") || args[0].startsWith("/")) { if(args[0].contains("v") || args[0].contains("version")) @@ -194,22 +219,22 @@ public static void main(String[] args) { { // archive flag handle if(args[0].contains("a")) archived = true; - + // secure request flag handle if(args[0].contains("s")) secured = true; - + // secure request flag handle if(args[0].contains("m")) runMac = true; - + if(archived || secured || runMac) { for(int i = 1; i < args.length; i++) { - System.out.println("File : " + args[i]); + System.out.println("File/Directory : " + args[i]); fileUrlListUp(args[i],archived,secured,runMac); } } - + } } else { @@ -220,9 +245,34 @@ public static void main(String[] args) { } } } - + endMessage(); } + + //recursively visit the directory/files and subfiles by Jossie + public static ArrayList visitFileRecursive(String path) throws IOException { + + ArrayList files = new ArrayList(); + + Files.walkFileTree(Paths.get(path), new SimpleFileVisitor() { + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + + files.add(file.toString()); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException{ + System.err.println(exc); + return FileVisitResult.CONTINUE; + } + }); + + return files; + } + }