Get All URLs From a Website Using Java Code

In this post, I am going to show you a Java class using that we can get all URLs from a website URL like a web crawler.

Here goes the Java Class

1. package client;
3. import java.io.IOException;
4. import java.io.InputStream;
5. import java.io.InputStreamReader;
6. import java.io.Reader;
8. import java.net.MalformedURLException;
9. import java.net.URI;
10. import java.net.URISyntaxException;
11. import java.net.URL;
13. import java.util.Enumeration;
15. import javax.swing.text.MutableAttributeSet;
16. import javax.swing.text.html.HTML;
17. import javax.swing.text.html.HTMLEditorKit;
18. import javax.swing.text.html.parser.ParserDelegator;
20. public class GetAllUrls {
21. public GetAllUrls() {
22. super();
23. }
25. public static void main(String[] args) {
26. final String name = "http://www.awasthiashish.com";
27. Reader r = null;
29. try {
30. URL u = new URL(name);
31. InputStream in = u.openStream();
32. r = new InputStreamReader(in);
34. ParserDelegator hp = new ParserDelegator();
35. hp.parse(r, new HTMLEditorKit.ParserCallback() {
37. public void handleStartTag(HTML.Tag t, MutableAttributeSet a, int pos) {
39. if (t == HTML.Tag.A) {
40. Enumeration attrNames = a.getAttributeNames();
42. while (attrNames.hasMoreElements()) {
43. String exactUrl = "";
44. Object key = attrNames.nextElement();
45. if ("href".equals(key.toString())) {
46. exactUrl = a.getAttribute(key).toString();
48. if (!a.getAttribute(key).toString().startsWith("http://") &&
49. !a.getAttribute(key).toString().startsWith("https://")) {
51. if (a.getAttribute(key).toString().startsWith("/")) {
52. exactUrl = name + a.getAttribute(key);
53. } else {
54. exactUrl = name.concat("/").concat(a.getAttribute(key).toString());
55. }
57. URI uri;
59. try {
60. uri = new java.net.URI(exactUrl);
61. System.out.println(uri);
62. } catch (URISyntaxException e) {
63. }
66. } else {
67. URI uri;
68. try {
69. uri = new java.net.URI(exactUrl);
70. System.out.println(uri);
71. } catch (URISyntaxException e) {
72. }
73. }
74. }
75. }
76. }
77. }
78. }, true);
79. } catch (MalformedURLException e) {
80. } catch (IOException e) {
81. } finally {
83. if (r != null) {
84. try {
85. r.close();
86. } catch (IOException e) {
87. }
88. }
89. }
90. }
91. }

and output on the console

Here you can see all the URLs linked to the home page of this website

Cheer 🙂 Happy Learning