Question

program is called grabing website data, to scrap all data from a website ( for example google ) with tabs (hyperlinks)and copying them in a file using java their is HTML code but i can't remeber it must be added and we must get these data in an array then we copy it in the file and please simply without using advanced java libraries , is there any help

Answer 1

Used no advance / third party libraries just basic code divided into multiple methods.

in main method find todo section and update

save the below code as GrabData.java

import java.io.BufferedReader;
import java.io.FileWriter;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class GrabData {

    /**
     * function to write links to text file
     * @param fileName
     * @param links
     */
    public static void writeTofFile(String fileName,List<String>links){
        try (FileWriter writer = new FileWriter(fileName)) {
            for(String link:links)
                writer.write(link+"\n");
            writer.flush();
        }catch (Exception e){
            e.printStackTrace();
            System.exit(0);
        }

    }

    /**
     * function to retrieve html htmldata from given url
     * @param theUrl
     * @return
     */
    public static String getUrlContents(String theUrl) {
        // empty stringbuilder for adding string data for html
        StringBuilder htmldata = new StringBuilder();
        try {
            // building url object  with given url
            URL url = new URL(theUrl);
            URLConnection connection = url.openConnection();
            BufferedReader bf = new BufferedReader(new InputStreamReader(connection.getInputStream()));
            String temp;
            // reading line by line html content
            while ((temp = bf.readLine()) != null) {
                htmldata.append(temp + "\n");
            }
            bf.close();
        }
        catch(Exception e) {
            e.printStackTrace();
            System.exit(0);
        }
        return htmldata.toString();
    }

    /**
     * function to retrieve all links from html page
     * @param HTMLPage
     * @return
     */
    public static ArrayList<String> getAllLinks( String HTMLPage){
        Pattern linkPat =  Pattern.compile("(<a[^>]+>.+?<\\/a>)",  Pattern.CASE_INSENSITIVE|Pattern.DOTALL);
        Matcher pageMatcher = linkPat.matcher(HTMLPage);
        // Array for adding all links
        ArrayList<String> links = new ArrayList<String>();
        while(pageMatcher.find()){
            String s =pageMatcher.group();
            if(s.contains("https://") || s.contains("http://")) {
                String link=s.split("href=")[1].split(">")[0].split("\"")[1];
                links.add(link);
            }
        }
        return links;
    }
    public static void main(String[] args) {
        // TODO: change the URL name as
        final String URL = "https://www.google.com";
        // TODO: change the file name as per needed
        final String fileName = "urls.txt";
        String htmlPage = getUrlContents(URL);
        List<String> links = getAllLinks(htmlPage);
        writeTofFile(fileName,links);
        System.out.println("OUTPUT WRITTEN TO "+fileName);
    }

}

//OUTPUT

OUTPUT WRITTEN TO urls.txt

a txt file will be generated that contains all urls

Please do comment in case of any concern...