AR4WE

#!/bin/bash

# Author: Zakaria Farahi & Saad Lili & Haraoui Souhaib
# Created: 10 May 2024
# Last modified: 10 May 2024

# Description: This script is used to do reconaissance on a target domain. It uses various tools to gather information about the target domain.

# Usage: AR4WE [options] <domain>
# Options:
# -h, --help: Display help message
# -V, --verbose: Display verbose output
# -t, --threads: Execute using threads

# 100 : exit code for help message
# 1 : exit code for not root user
# 101 : exit code for invalid option
# 102 : exit code for domain not provided or invalid

clear

echo "
      .o.       ooooooooo.         .o   oooooo   oooooo     oooo oooooooooooo 
     .888.      \`888   \`Y88.     .d88    \`888.    \`888.     .8'  \`888'     \`8 
    .8\"888.      888   .d88'   .d'888     \`888.   .8888.   .8'    888         
   .8' \`888.     888ooo88P'  .d'  888      \`888  .8'\`888. .8'     888oooo8    
  .88ooo8888.    888\`88b.    88ooo888oo     \`888.8'  \`888.8'      888    \"    
 .8'     \`888.   888  \`88b.       888        \`888'    \`888'       888       o 
o88o     o8888o o888o  o888o     o888o        \`8'      \`8'       o888ooooood8 
                                                                          
"

optionHelp() {
    echo "Usage: $0 [options] <domain>"
    echo "Options:"
    echo "  -V            Display verbose output"
    echo "  -h            Show help message"
    echo "  -t <threads>  Specify number of threads"
    echo "  -c <step>     Continue from a specific step (1-10)"
}

# Check if the first argument is help or empty
if [ "$1" == "-h" ] || [ "$1" == "--help" ] || [ -z "$1" ]; then
    optionHelp
    exit 100
fi

# Check if running as root
if [ "$EUID" -ne 0 ]; then
    echo "Please run as root"
    exit 1
fi



if [ "$(id -u)" -eq 0 ]; then
    SUDO=""
else
    SUDO="sudo"
fi
# Variables to store options
verbose="false"
threads=1
etape=1

# Parse options
while getopts ":Vht:c:" option; do
    case $option in
    V) verbose="true" ;;
    h)
        optionHelp
        exit 0
        ;;
    t)
        # Number of threads it may be float
        if [[ $OPTARG =~ ^[0-9]+$ ]]; then
            threads=$OPTARG
            if ! command -v bc &>/dev/null; then
                apt install bc -y
            fi
        else
            echo "Option -t requires a numeric argument."
            exit 101
        fi
        ;;
    c)
        # continue from etape
        if [[ $OPTARG =~ ^[0-9]+$ ]]; then
            etape=$OPTARG
        else
            echo "Option -c requires a numeric argument."
            echo "1 : Domain Information"
            echo "2 : Subdomain Enumeration"
            echo "3 : Live Subdomains"
            echo "4 : Port Scanning"
            echo "5 : Content Discovery"
            echo "6 : Sub-subdomains Enumeration"
            echo "7 : Waybackurls"
            echo "8 : gau"
            echo "9 : gf"
            exit 101
        fi
        ;;
    \?)
        echo "Invalid option: -$OPTARG" >&2
        exit 101
        ;;
    esac
done

# Shift the parsed options
shift $((OPTIND - 1))

# Get the domain
domain="$1"

# Check if domain is provided
if [ -z "$domain" ] || ! [[ $domain =~ ^.+\..+$ ]]; then
    echo "Error: Domain not provided or invalid. Please provide a valid domain."
    exit 102
fi

# Use the domain variable as needed
echo "Domain: $domain"
echo "Verbose: $verbose"



# start
echo "Starting reconaissance on $domain"

# Create a directories to store the results

if [ ! -d "$domain" ]; then
    echo "INFOS : Creating directory with name $domain..."
    mkdir -p $domain
else
    echo "INFOS : Directory already exists"
    if [[ $etape -eq 1 ]]; then
        echo "INFOS : Continuing from the first step"
        # compress backup the directory if it already exists then delete its content
        date=$(date '+%Y-%m-%d_%H-%M-%S')
        echo "INFOS : Compressing the directory..."
        tar -czvf $domain-$date.tar.gz $domain >/dev/null 2>&1
        rm -rf $domain/*
    else
        echo "INFOS : Continuing from step $etape"
    fi
fi

cd $domain

if [[ $etape -eq 1 ]]; then
    mkdir -p Domain_Info lastPart/backurl/gau lastPart/backurl/wayback lastPart/contentDir/DirSearch lastPart/contentDir/fuzz lastPart/JsFiles lastPart/subSub/AltDNS liveSubDom/httpx port_Scan/masscan port_Scan/nmap Subdomain/Amass Subdomain/Sublist3r Subdomain/Subfinder logs screenshots/ 
    touch Domain_Info/DNS_Rec.txt Domain_Info/whois.txt lastPart/backurl/gau/res_gau.txt lastPart/backurl/res_final.txt lastPart/backurl/res_way.txt lastPart/contentDir/DirSearch/res_Dir.txt lastPart/contentDir/fuzz/res_f.txt lastPart/contentDir/res_final.txt lastPart/JsFiles/js_url.txt lastPart/subSub/AltDNS/res.txt liveSubDom/httpx/live.txt port_Scan/masscan/res_mass.txt port_Scan/nmap/res_nmap.txt Subdomain/Amass/res_amass.txt Subdomain/ip_add.txt Subdomain/subdomains_filterd.txt Subdomain/Sublist3r/res_Sub.txt Subdomain/Subfinder/res_subfinder.txt port_Scan/masscan/ips port_Scan/masscan/temp port_Scan/masscan/open_ports logs/logs_file.txt lastPart/res_final.txt


# Define the project and spider names
project_name="urlsJsScraper"
# Function to set up Scrapy project and spiders
setup_scrapy_project() {
    # Create Scrapy project
    if [ ! -d "/venv" ]; then
			python3 -m venv /venv >/dev/null
		fi
		source /venv/bin/activate
    scrapy startproject urlsJsScraper >/dev/null 2>&1
    if [ $? -eq 0 ]; then # Check if scrapy command succeeded
        mkdir -p lastPart
        mv urlsJsScraper lastPart/
        cd "./lastPart/urlsJsScraper"
    python3 -m venv venv
    source venv/bin/activate
    pip install scrapy >/dev/null 2>&1
        # Write the spider code to a new file in the spiders directory
        cat <<EOF >"urlsJsScraper/spiders/url_extractor.py"
import scrapy

class UrlExtractorSpider(scrapy.Spider):
    name = 'url_extractor'

    def __init__(self, subdomains_file=None, *args, **kwargs):
        super(UrlExtractorSpider, self).__init__(*args, **kwargs)
        self.subdomains_file = subdomains_file

    def start_requests(self):
        if self.subdomains_file:
            with open(self.subdomains_file, "r") as file:
                urls = [url.strip() for url in file.readlines() if url.strip()]
            for url in urls:
                if not url.startswith(('http://', 'https://')):
                    url = 'http://' + url
                yield scrapy.Request(url=url, callback=self.parse)
        else:
            self.logger.error("Subdomains file not provided.")
            return  # Stop the spider if no file is provided

    def parse(self, response):
        # Extract all links and JS file paths
        for link in response.css('a::attr(href)').getall():
            yield {'url': response.urljoin(link)}
        for script in response.css('script::attr(src)').getall():
            if script:  # Check if the src attribute is not empty
                yield {'js_path': response.urljoin(script)}
EOF
        cd ../../
    else
        echo "Failed to create project directory urlsJsScraper"
        return 1
    fi
}
# call setup scrapy prject
setup_scrapy_project
fi
log_message() {
    local log_level=$1
    local message=$2

    # Log to the file
    echo "$(date '+%Y-%m-%d-%H-%M-%S') : $(whoami) : $log_level : $message" >>logs/logs_file.txt

    # Print to console if verbose mode is enabled
    if [ "$verbose" == "true" ]; then
        echo "$message"
    fi
}

# Domain Information
#####################################
domainInfo() {
    # For INFO log
    log_message "INFOS" "Starting domain information gathering with whois"
    whois "$domain" >Domain_Info/whois.txt >/dev/null 2>&1
    if [ $? -ne 0 ]; then
        log_message "ERROR" "We have some problems with whois"
        domainCheck="false"
    fi
    log_message "INFOS" "The end of whois we successfully gathered $(wc -l Domain_Info/whois.txt) lines of information"
    log_message "INFOS" "Starting domain information gathering with nslookup"
    nslookup "$domain" >Domain_Info/DNS_Rec.txt >/dev/null 2>&1
    if [ $? -ne 0 ]; then
        log_message "ERROR" "We have some problems with nslookup"
        domainCheck="false"
    fi
    log_message "INFOS" "The end of nslookup we successfully gathered $(wc -l Domain_Info/DNS_Rec.txt) lines of information"
}

# Call
callDomainInfo() {
    domainCheck="true"
    domainInfo
    if [ "$domainCheck" == "false" ]; then
    log_message "ERROR" "lDomain Information failed"
        domainInfo
        if [ "$domainCheck" == "false" ]; then
            log_message "ERROR" "Error: Domain Information failed"
            log_message "INFOS" "Continuing with the rest of the script"
        fi
    fi
}

#####################################
# Subdomain Enumeration
subEnum() {
    log_message "INFOS" "Starting subdomain enumeration with sublist3r"
    source /opt/Sublist3r/venv/bin/activate
    sublist3r -d $domain -o Subdomain/Sublist3r/res_Sub.txt -t $(echo "scale=5; $threads * 16" | bc) >/dev/null 2>&1
    if [ $? -ne 0 ]; then
        log_message "ERROR" "We have some problems sublist3r"
        subCheck="false"
    fi
    log_message "INFOS" "The end of sublist3r we successfully gathered $(wc -l Subdomain/Sublist3r/res_Sub.txt) lines of information"

    log_message "INFOS" "Starting subdomain enumeration with amass"
    amass enum -d $domain -o Subdomain/Amass/res_amass.txt -silent >/dev/null 2>&1
    if [ $? -ne 0 ]; then
    log_message "ERROR" "We have some problems with amass"
        subCheck="false"
    else
        grep -Eo "([0-9]{1,3}\.){3}[0-9]{1,3}" Subdomain/Amass/res_amass.txt >Subdomain/ip_add.txt
        grep -Eo "([a-z0-9A-Z]+\.[a-z0-9A-Z]+\.?[a-z0-9A-Z]+)" Subdomain/Amass/res_amass.txt >Subdomain/Amass/res_amass.txt >/dev/null 2>&1

    fi
    log_message "INFOS" "The end of amass we successfully gathered $(wc -l Subdomain/Amass/res_amass.txt) lines of information"

    log_message "INFOS" "Starting subdomain enumeration with subfinder"
    subfinder -d $domain -o Subdomain/Subfinder/res_subfinder.txt >/dev/null 2>&1
    if [ $? -ne 0 ]; then
        log_message "ERROR" "We have some problems with subfinder"
        subCheck="false"
    fi
    log_message "INFOS" "The end of subfinder we successfully gathered $(wc -l Subdomain/Subfinder/res_subfinder.txt) lines of information"

    # merge the resulates of sublist3r and amass
    log_message "INFOS" "start of merge the resulates of sublist3r and amass and subfinder"
    cat Subdomain/Sublist3r/res_Sub.txt Subdomain/Amass/res_amass.txt Subdomain/Subfinder/res_subfinder.txt >Subdomain/subdomains_filterd.txt
    log_message "INFOS" "end of merge the resulates of sublist3r and amass and subfinder and we successfully gathered $(wc -l Subdomain/subdomains_filterd.txt) lines of information"

    # get the ip addresses of the subdomains
    log_message "INFOS" "start of getting ip addresses"
    while read subdomain; do
        echo $subdomain
        dig +short "$subdomain" | awk '/([0-9]{1,3}\.){3}[0-9]{1,3}/ { print $1 }' >> Subdomain/ip_add.txt
    done <Subdomain/subdomains_filterd.txt
    # remove duplicates ip
    sort -u Subdomain/ip_add.txt -o Subdomain/ip_add.txt
    # remove last line from ip_add.txt
    sed -i '$ d' Subdomain/ip_add.txt
    log_message "INFOS" "end of getting ip addresses and we successfully gathered $(wc -l Subdomain/ip_add.txt) lines of information"
}
# Call
callSubEnum() {
    subCheck="true"
    subEnum
    if [ "$subCheck" == "false" ]; then
        log_message "ERROR" "Subdomain Enumeration failed"
        subEnum
        if [ "$subCheck" == "false" ]; then
            log_message "ERROR" "Subdomain Enumeration failed"
            log_message "INFOS" "Continuing with the rest of the script"
        fi
    fi
}
#####################################
# live subdomains
# live subdomains
liveSub() {
    log_message "INFOS" "Start of httpx"
    httpx -l Subdomain/subdomains_filterd.txt -o liveSubDom/httpx/live.txt -t $(echo "scale=5; $threads * 50" | bc) >/dev/null 2>&1
    if [ $? -ne 0 ]; then
        log_message "ERROR" "We have some problems with httpx"
        liveCheck="false"
    fi
    log_message "INFOS" "End of httpx we successfully gathered $(wc -l liveSubDom/httpx/live.txt) lines of information"
}

# Call
callLiveSub() {
    liveCheck="true"
    liveSub
    if [ "$liveCheck" == "false" ]; then
        log_message "ERROR" "Live subdomains failed"
        liveSub
        if [ "$liveCheck" == "false" ]; then
            log_message "ERROR" "Live subdomains failed"
            log_message "INFOS" "Continuing with the rest of the script"
        fi
    fi
}

#####################################
# Port Scanning
portScan() {
    log_message "INFOS" "Start of masscan"
    $SUDO masscan -p0-79,81-442,444-65535 -iL Subdomain/ip_add.txt --rate=10000 -oB port_Scan/masscan/temp
    $SUDO masscan --readscan port_Scan/masscan/temp | awk '{print $NF":"$4}' | cut -d/ -f1 >port_Scan/masscan/open_ports
    if [ $? -ne 0 ]; then
        log_message "ERROR" "We have some problems with masscan"
        portCheck="false"
    fi
    log_message "INFOS" "End of masscan we successfully gathered $(wc -l port_Scan/masscan/open_ports) lines of information"

    log_message "INFOS" "Start of nmap"
    nmap -sV -iL Subdomain/ip_add.txt -oN port_Scan/nmap/res_nmap.txt #>/dev/null 2>&1
    if [ $? -ne 0 ]; then
        log_message "ERROR" "We have some problems with nmap"
        portCheck="false"
    else
        log_message "INFOS" "End of nmap with success we successfully gathered $(wc -l port_Scan/nmap/res_nmap.txt) lines of information"
    fi
}

# Call
callPortScan() {
    portCheck="true"
    portScan
    if [ "$portCheck" == "false" ]; then
        log_message "ERROR" "Port Scanning failed"
        portScan
        if [ "$portCheck" == "false" ]; then
            log_message "ERROR" "Port Scanning failed"
            log_message "INFOS" "Continuing with the rest of the script"
        fi
    fi
}

#####################################
# Content Discovery
contentDis() {
    log_message "INFOS" "Start of DirSearch"
     dirsearch -u https://$domain -e php,asp,aspx,jsp,html,zip,jar,war,txt,log,conf,config,backup,old,sql,db,xml,svg,js,css -w wordlists/dirb/common.txt -o lastPart/contentDir/DirSearch/res_Dir.txt -t $(echo "scale=5; $threads * 25" | bc) >/dev/null 2>&1
     #while read subdomain; do
        #dirsearch -u https://$subdomain -e .php,.asp,.aspx,.jsp,.html,.zip,.jar,.war,.txt,.log,.conf,.config,.backup,.old,.sql,.db,.xml,.svg,.js,.css -w wordlists/dirb/common.txt -o lastPart/contentDir/DirSearch/res_Dir_$subdomain.txt
     #done < Subdomain/subdomains_filterd.txt
     if [ $? -ne 0 ]; then
         log_message "ERROR" "DirSearch failed"
         contentCheck="false"
     fi
    log_message "INFOS" "End of DirSearch we successfully gathered $(wc -l lastPart/contentDir/DirSearch/res_Dir.txt) lines of information"

    log_message "INFOS" "Start of ffuf"
     ffuf -w /opt/wordlists/dirb/common.txt -u https://$domain/FUZZ -e .php,.asp,.aspx,.jsp,.html,.zip,.jar,.war,.txt,.log,.conf,.config,.backup,.old,.sql,.db,.xml,.svg,.js,.css -o lastPart/contentDir/fuzz/res_f.txt -t $(echo "scale=5; $threads * 40" | bc) >/dev/null 2>&1
     #while read subdomain; do
         #ffuf -w /opt/wordlists/dirb/common.txt -u https://$subdomain/FUZZ -e .php,.asp,.aspx,.jsp,.html,.zip,.jar,.war,.txt,.log,.conf,.config,.backup,.old,.sql,.db,.xml,.svg,.js,.css -o lastPart/contentDir/fuzz/res_f_$subdomain.txt >/dev/null 2>&1
     #done <Subdomain/subdomains_filterd.txt
     if [ $? -ne 0 ]; then
         log_message "ERROR" "We have some problems with ffuf"
         contentCheck="false"
     fi
    log_message "INFOS" "End of ffuf we successfully gathered $(wc -l lastPart/contentDir/fuzz/res_f.txt) lines of information"

    cat lastPart/contentDir/DirSearch/res_Dir.txt lastPart/contentDir/fuzz/res_f.txt >lastPart/contentDir/res_final.txt
}

# Call
callContentDis() {
    contentCheck="true"
    #contentDis
    if [ "$contentCheck" == "false" ]; then
        log_message "ERROR" "Content Discovery failed"
        contentDis
        if [ "$contentCheck" == "false" ]; then
            log_message "ERROR" "Content Discovery failed"
            log_message "INFOS" "Continuing with the rest of the script"
        fi
    fi
}

#####################################
## sub-subdomains enumeration
subSubEnum() {
    log_message "INFOS" "Start of altdns"
    altdns -i Subdomain/subdomains_filterd.txt -o lastPart/subSub/res.txt -w /opt/wordlists/subSub.txt -r -s lastPart/subSub/resolved.txt >/dev/null 2>&1
    if [ $? -ne 0 ]; then
        log_message "ERROR" "We have some problems with altdns"
        subSubCheck="false"
    else
        log_message "INFOS" "End of altdns with success we successfully gathered $(wc -l lastPart/subSub/res.txt) lines of information"
    fi
}

# Call
callSubSubEnum() {
    subSubCheck="true"
    subSubEnum
    if [ "$subSubCheck" == "false" ]; then
        log_message "ERROR" "Sub-subdomains enumeration failed"
        subSubEnum
        if [ "$subSubCheck" == "false" ]; then
            log_message "ERROR" "Sub-subdomains enumeration failed"
            log_message "INFOS" "Continuing with the rest of the script"
        fi
    fi
}

#####################################
# waybackurls
Execwayback() {
    log_message "INFOS" "Start of waybackurls"
    cat Subdomain/subdomains_filterd.txt | waybackurls >lastPart/backurl/wayback/res_way.txt
    if [ $? -ne 0 ]; then
        log_message "ERROR" "We have some problems with waybackurls"
        waybackCheck="false"
    else
        log_message "INFOS" "End of waybackurls with success we successfully gathered $(wc -l lastPart/backurl/wayback/res_way.txt) lines of information"
    fi
    cat lastPart/contentDir/res_final.txt lastPart/backurl/wayback/res_way.txt >lastPart/res_final.txt
}

# Call
callWayback() {
    waybackCheck="true"
    Execwayback
    if [ "$waybackCheck" == "false" ]; then
        log_message "ERROR" "waybackurls failed"
        Execwayback
        if [ "$waybackCheck" == "false" ]; then
            log_message "ERROR" "waybackurls failed"
            log_message "INFOS" "Continuing with the rest of the script"
        fi
    fi
}

# Call wayback function

#####################################

# gau
execgau() {
    log_message "INFOS" "Start of gau"
    cat Subdomain/subdomains_filterd.txt | gau -o lastPart/backurl/gau/res_gau.txt >/dev/null 2>&1
    if [ $? -ne 0 ]; then
        log_message "ERROR" "We have some problems with gau"
        gauCheck="false"
    else
        log_message "INFOS" "End of gau we successfully gathered $(wc -l lastPart/backurl/gau/res_gau.txt) lines of information"
    fi
}

# Call
callGau() {
    gauCheck="true"
    execgau
    if [ "$gauCheck" == "false" ]; then
        log_message "ERROR" "gau failed"
        execgau
        if [ "$gauCheck" == "false" ]; then
            log_message "ERROR" "gau failed"
            log_message "INFOS" "Continuing with the rest of the script"
        fi
    fi
}

# Call gau function

#####################################
# gf
execgf() {
    log_message "INFOS" "Start of gf"
    cat lastPart/res_final.txt | gf xss | tee -a lastPart/xssgf.txt
    cat lastPart/res_final.txt | gf ssti | tee -a lastPart/sstigf.txt
    cat lastPart/res_final.txt | gf sqli | tee -a lastPart/sqligf.txt
    cat lastPart/res_final.txt | gf ssrf | tee -a lastPart/ssrfgf.txt
    cat lastPart/res_final.txt | gf redirect | tee -a lastPart/redirectgf.txt
    if [ $? -ne 0 ]; then
        log_message "ERROR" "We have some problems with gf"
        gfCheck="false"
    else
        log_message "INFOS" "End of gf with success"
    fi
}

# Call
callGf() {
    gfCheck="true"
    execgf
    if [ "$gfCheck" == "false" ]; then
        log_message "ERROR" "gf failed"
        execgf
        if [ "$gfCheck" == "false" ]; then
            log_message "ERROR" "gf failed"
            log_message "INFOS" "Continuing with the rest of the script"
        fi
    fi
}

# Call gf function

#####################################

# scrapy
scrapy_func() {
    log_message "INFOS" "Start of Extracting urls & js paths"
    (cd lastPart/urlsJsScraper && scrapy crawl url_extractor -a subdomains_file=../../Subdomain/subdomains_filterd.txt -o ../UrlsJsPaths/results.json >/dev/null 2>&1)
    if [ $? -ne 0 ]; then
        log_message "ERROR" "We have some problems with Extracting urls & js paths"
        scrapyCheck="false"
    else
        log_message "INFOS" "End of Extracting urls & js paths with success we successfully gathered $(wc -l lastPart/urlsJsScraper/results.json) lines of information"
    fi
}

# call
callScrapyFunc() {
    scrapyCheck="true"
    scrapy_func
    if [ "$scrapyCheck" == "false" ]; then
        log_message "ERROR" "scrapy failed"
        log_message "INFOS" "Continuing with the rest of the script"
    fi
}

# Call scrapy function


execute_steps_from() {
    step=$1
    case "$step" in
        1) callDomainInfo;;
        2) callSubEnum;;
        3) callLiveSub;;
        4) callPortScan;;
        5) callContentDis;;
        6) callSubSubEnum;;
        7) callWayback;;
        8) callGau;;
        9) callGf;;
        10) callScrapyFunc;;
        *) echo "Invalid step"; exit 1;;
    esac
}

main() {
    step=$etape

    while [ $step -le 10 ]; do
        execute_steps_from $step
        step=$((step + 1))
    done
}

main