search

mmiller7 / Netgear_modem_scrape

2 stars 3 forks source link

How do i install this? #1

Open DeFlanko opened 2 years ago

DeFlanko commented 2 years ago

How do i install this?

DeFlanko commented 9 months ago

@mmiller7 Ok jumped the gun with earlier responses -- i had to re orientate myself... (its been a crazy busy year and im sorry im just now getting back to this)

So first things first was to absorb what you said about the echos everywhere... so i did that here:

#!/bin/bash

# Default mqtt_password is "password"
modem_username="admin"
modem_password=#########
modem_ip="192.168.100.1"
baseURL="https://192.168.100.1"

# Settings for MQTT mqtt_broker to publish stats
mqtt_broker="172.10.1.12"
#mqtt_username="Admin"
#mqtt_password=##########

mqtt_username="homeassistant"
mqtt_password=################
mqtt_topic="homeassistant/sensor/modemsignals"

# HomeAssistant doesn't expose this to the container so we have to hack it up
# Comment these out for a "normal" host that knows where mosquitto_pub is on its own
export LD_LIBRARY_PATH='/config/bin/mosquitto_deps/lib'
mqtt_pub_exe="/config/bin/mosquitto_deps/mosquitto_pub"
# Uncomment tthis for a "normal" host that knows where mosquitto_pub is on its own
#mqtt_pub_exe="mosquitto_pub"
#echo "line 25"
# Cookie file path
cookie_path="$0.cookie"

#####################################
# Prep functions to interface modem #
#####################################

# This function publishes login status helpful for debugging
#echo "line 34"

function loginStatus () {
    #echo "Modem login: $1"
    # Publish MQTT to announce status
    if [ "$2" != "" ]; then
        message="{ \"login\": \"$1\", \"detail\": \"$2\" }"
    else
        message="{ \"login\": \"$1\" }"
    fi
    $mqtt_pub_exe -h "$mqtt_broker" -u "$mqtt_username" -P "$mqtt_password" -t "${mqtt_topic}/login" -m "$message" || echo "MQTT-Pub Error!"
}
#echo "line 45"
# Fetch the page and then print the result
webToken=""
realSessionId=""
lastPageFetch=""

echo "line 50"
function pageDive () {
  toFetch="$1"
  referPage="$2"
  depth=$(( $depth + 1 ))
  lastPageFetch="$toFetch"

  if [[ "$depth" -ge 5 ]]; then
    loginStatus "max_redirect_exceeded"
  else

      # Fetch the page
      if [ "$referPage" != "" ]; then
        referURL="${baseURL}${referPage}"
      fi

        # Decide which request to make (if page requires post data)
        if [ "$toFetch" == "/goform/GenieLogin" ]; then
            data=$(curl --insecure --connect-timeout 5 -v -s -e "$referURL" -b "$cookie_path" -c "$cookie_path" -X POST -H 'Content-Type: application/x-www-form-urlencoded' --data-urlencode "loginUsername=$modem_username" --data-urlencode "loginPassword=$modem_password" --data "login=1" --data "webToken=$webToken" "${baseURL}${toFetch}"  2>&1)
            exitCode=$?
        elif [ "$toFetch" == "/goform/MultiLogin" ]; then
            data=$(curl --insecure --connect-timeout 5 -v -s -e "$referURL" -b "$cookie_path" -c "$cookie_path" -X POST -H 'Content-Type: application/x-www-form-urlencoded' --data "yes=yes" --data "Act=yes" --data "RetailSessionId=$retailSessionId" --data "webToken=$webToken" "${baseURL}${toFetch}"  2>&1)
            exitCode=$?
        else
            #data=$(curl --insecure --connect-timeout 10 -v -s -e "$referURL" -b "$cookie_path" -c "$cookie_path" "${baseURL}${toFetch}" 2>&1)
            data=$(curl --insecure --connect-timeout 10 -v -s -e -i "$referURL" -b "$cookie_path" -c "$cookie_path" -F "loginUsername=$modem_username;loginPassword=$modem_password" "${baseURL}${toFetch}" 2>&1)
            exitCode=$?
        fi

      # Check for timeout and decide what to do next
      if [ "$exitCode" == 28 ]; then 
        loginStatus "failed_timeout" "${toFetch}"
        exit 11
      fi

      # Get redirect location if any
      redirectPage=$(echo "$data" | egrep 'window.top.location|< Location: ' | awk '{print $3}' | sed 's/[";]//g')

      # Pull out hidden fields if applicable
      webToken=$(echo "$data" | egrep -o 'name="webToken" value=[0-9]+' | egrep -o '[0-9]+')
      retailSessionId=$(echo "$data" | egrep -o 'name=\"RetailSessionId\" value=\"[^\"]*\"' | awk -F "\"" '{print $4}')

      # Decide if we need to submit a form and redirect
      #loginStatus "toFetch=$toFetch"
      if [ "$toFetch" == "/GenieLogin.asp" ]; then
        redirectPage="/goform/GenieLogin"
      elif [ "$toFetch" == "/MultiLogin.asp" ]; then
        redirectPage="/goform/MultiLogin"
      fi

      # Drill down if applicable
      if [ "$redirectPage" == "" ]; then
        echo "$data"
      else
        loginStatus "redirect" "${redirectPage}"
        pageDive "$redirectPage" "$toFetch"
      fi
  fi
  depth=$(( $depth - 1 ))
}
echo "line 117"

# This function gets a dynamic session-cookie from the modem
function getSession () {

    # Perform the login, or verify we are already logged in
    pageDive > /dev/null 

}
echo "line 128"
# This function fetches the HTML status page from the modem for parsing
function getResult () {
# Finally, we can request the page

    result=$(pageDive '/DocsisStatus.asp' 'GenieIndex.asp')

    # If we were given GenieIndex instead of DocsisStatus, that means it went thru a redirect
    # so we will try exactly one more time.
    if [ "$(echo "$result" | grep -c '> GET /GenieIndex.asp HTTP/1.1')" == "1" ] ; then
        result=$(pageDive '/DocsisStatus.asp' 'GenieIndex.asp')
    fi

}
echo "line 143"

#############################
# Log in and fetch the data #
#############################

# Get the result from the modem
#getSession;
getResult;

# See if we were successful
if [ "$(echo "$result" | grep -c 'Downstream Bonded Channels')" == "0" ] ; then
    loginStatus "failed_retrying"

#   # If we failed (got a login prompt) try once more for new token
    rm "$cookie_path"
    pageDive '/MultiLogin.asp' > /dev/null
    pageDive '/Logout.asp' > /dev/null
    pageDive '/Logout.asp' > /dev/null
    rm "$cookie_path"

    getSession;
    getResult;
fi
echo "line 168"

# See if we were successful
if [ "$(echo "$result" | grep -c 'Downstream Bonded Channels')" == "0" ] ; then
    # At this point, if we weren't successful, we give up
    loginStatus "failed"
    exit 21
else
    loginStatus "success"
    #echo "$result"
fi

echo "line 178"
# Log out afterward
#pageDive '/Logout.asp' > /dev/null

####################
# Parse the result #
####################

#echo "Raw:"
#echo -e "$result"

#echo "$result" | tr '\n' ' ' | sed 's/\t//g;s/ //g;s/dBmV//g;s/dB//g' | awk -F "<tableclass=['\"]simpleTable['\"]>|</table>" '{print "\nStartup:\n" $2 "\n\nDown\n" $4 "\n\nUp\n" $6 "\n" }'
#startup_status=$(echo "$result" | tr '\n' ' ' | sed 's/\t//g;s/ //g;s/dBmV//g;s/dB//g;s/Hz//g;s/<[/]\?strong>//g;s/<![^>]*>//g;s/<[/]\?[bui]>//g' | awk -F "tableid=\"startup_procedure_table\"" '{print $2}'  )
startup_status=$(echo "$result" | tr '\n\r' ' ' | sed 's/\t//g;s/ //g;s/<!--Hiddenconfigfilefield-->//g' | awk -F "tableid=\"startup_procedure_table\"" '{print $2}' | awk -F "</table>" '{print $1}')
downstream_status=$(echo "$result" | tr '\n' ' ' | sed 's/\t//g;s/ //g;s/dBmV//g;s/dB//g;s/Hz//g;s/<[/]\?strong>//g;s/<![^>]*>//g' | awk -F "<[/]?tabindex=-1>" '{print $5}' | awk -F "</table>" '{print $1}')
upstream_status=$(echo "$result" | tr '\n' ' ' | sed 's/\t//g;s/ //g;s/dBmV//g;s/dB//g;s/Hz//g;s/<[/]\?strong>//g;s/<![^>]*>//g' | awk -F "<[/]?tabindex=-1>" '{print $7}' | awk -F "</table>" '{print $1}')
ofdm_downstream_status=$(echo "$result" | tr '\n' ' ' | sed 's/\t//g;s/ //g;s/dBmV//g;s/dB//g;s/Hz//g;s/<[/]\?strong>//g;s/<![^>]*>//g' | awk -F "<[/]?tabindex=-1>" '{print $9}' | awk -F "</table>" '{print $1}')
ofdm_upstream_status=$(echo "$result" | tr '\n' ' ' | sed 's/\t//g;s/ //g;s/dBmV//g;s/dB//g;s/Hz//g;s/<[/]\?strong>//g;s/<![^>]*>//g' | awk -F "<[/]?tabindex=-1>" '{print $11}' | awk -F "</table>" '{print $1}')
system_up_time=$(echo "$result" | grep "SystemUpTime" | awk -F "</b>|</font>" '{print $2}')

# Break out by line
startup_rows=$(echo "$startup_status" | sed 's/^<tr>//g;s/<\/tr>$//g;s/<\/tr><tr[^>]*>/\n/g')
downstream_rows=$(echo "$downstream_status" | sed 's/^<tr>//g;s/<\/tr>$//g;s/<\/tr><tr[^>]*>/\n/g')
upstream_rows=$(echo "$upstream_status" | sed 's/^<tr>//g;s/<\/tr>$//g;s/<\/tr><tr[^>]*>/\n/g')
ofdm_downstream_rows=$(echo "$ofdm_downstream_status" | sed 's/^<tr>//g;s/<\/tr>$//g;s/<\/tr><tr[^>]*>/\n/g')
ofdm_upstream_rows=$(echo "$ofdm_upstream_status" | sed 's/^<tr>//g;s/<\/tr>$//g;s/<\/tr><tr[^>]*>/\n/g')
# Note: system_up_time is a single value and does not need additional parsing

# Break out columns

# Parse out the startup status HTML table into JSON and publish
#echo "$startup_rows"
#echo "$startup_rows" | sed 's/<th[^>]*>[^<]*<\/th>//g;s/^<td[^>]*>//g;s/<\/td>$//g;s/<\/td><td[^>]*>/\t/g' | grep -v "^$"
# Helper function to more easily build JSON per field
echo "line 214"
function pubStartupStatusValue () {
    # Break out field information
    procedure_name="$1"
    procedure_status="$2"
    procedure_comment="$3"
    # Build the message payload
    message=""
    # If exists, insert stattus
    if [ "$procedure_status" != "" ]; then
        if [[ "$procedure_status" =~ ^[0-9]+$ ]]; then
            message="${message} \"status\": $procedure_status"
        else
            message="${message} \"status\": \"$procedure_status\""
        fi
    fi
    # If exists, insert comment
    if [ "$procedure_comment" != "" ]; then
        # If message is not empty, insert separator comma
        if [ "$message" != "" ]; then
            message="${message}, "
        fi
        message="${message} \"comment\": \"$procedure_comment\""
    fi
    message="{ ${message} }"
    $mqtt_pub_exe -h "$mqtt_broker" -u "$mqtt_username" -P "$mqtt_password" -t "${mqtt_topic}/startup_procedure/${procedure_name}" -m "$message"
}
echo "$startup_rows" | grep -v "^$" | tail -n +2 | while read -r line; do
    to_parse=$(echo "$line" | sed 's/<th[^>]*>[^<]*<\/th>//g;s/^<td[^>]*>//g;s/<\/td>$//g;s/<\/td><td[^>]*>/\t/g')
    procedure_name=$(echo $to_parse | awk '{print $1}')
    procedure_status=$(echo $to_parse | awk '{print $2}')
    procedure_comment=$(echo $to_parse | awk '{print $3}')
    pubStartupStatusValue "$procedure_name" "$procedure_status" "$procedure_comment"
done

echo "line 249"

# Parse out the downstream HTML table into JSON and publish
#echo "$downstream_rows" | sed 's/<th[^>]*>[^<]*<\/th><\/tr>//g;s/^<td>//g;s/<\/td>$//g;s/<\/td><td[^>]*>/\t/g'
counter=0
echo "$downstream_rows" | tail -n +3 | while read -r line; do
    counter=$(($counter+1))
    #echo "${mqtt_topic}/downstream/$counter"
    to_parse=$(echo "$line" | sed 's/<th[^>]*>[^<]*<\/th><\/tr>//g;s/^<td>//g;s/<\/td>$//g;s/<\/td><td[^>]*>/\t/g')
    message=$(
        echo "{"
        echo "$to_parse" | awk '{ print "\"Channel\": "$1","
                                                            print "\"LockStatus\": \""$2"\","
                                                            print "\"Modulation\": \""$3"\","
                                                            print "\"ChannelID\": "$4","
                                                            print "\"Frequency\": "$5","
                                                            print "\"Power\": "$6","
                                                            print "\"SNR_MER\" :"$7","
                                                            print "\"Unerrored\" :"$8","
                                                            print "\"Corrected\" :"$9","
                                                            print "\"Uncorrectable\" :"$10 }'
        echo "}"
    )
    $mqtt_pub_exe -h "$mqtt_broker" -u "$mqtt_username" -P "$mqtt_password" -t "${mqtt_topic}/downstream/${counter}" -m "$message"
done

# Parse out the upstream HTML table into JSON and publish
#echo "$upstream_rows" | sed 's/<th[^>]*>[^<]*<\/th><\/tr>//g;s/^<td>//g;s/<\/td>$//g;s/<\/td><td[^>]*>/\t/g'
counter=0
echo "$upstream_rows" | tail -n +3 | while read -r line; do
    counter=$(($counter+1))
    #echo "${mqtt_topic}/upstream/$counter"
    to_parse=$(echo "$line" | sed 's/<th[^>]*>[^<]*<\/th><\/tr>//g;s/^<td>//g;s/<\/td>$//g;s/<\/td><td[^>]*>/\t/g')
    message=$(
        echo "{"
        echo "$to_parse" | awk '{ print "\"Channel\": "$1","
                                                            print "\"LockStatus\": \""$2"\","
                                                            print "\"Modulation\": \""$3"\","
                                                            print "\"ChannelID\": "$4","
                                                            print "\"Frequency\": "$5","
                                                            print "\"Power\": "$6 }'
        echo "}"
    )
    $mqtt_pub_exe -h "$mqtt_broker" -u "$mqtt_username" -P "$mqtt_password" -t "${mqtt_topic}/upstream/${counter}" -m "$message"
done

# Parse out the OFDM downstream HTML table into JSON and publish
counter=0
echo "$ofdm_downstream_rows" | tail -n +3 | while read -r line; do
    counter=$(($counter+1))
    to_parse=$(echo "$line" | sed 's/<th[^>]*>[^<]*<\/th><\/tr>//g;s/^<td>//g;s/<\/td>$//g;s/<\/td><td[^>]*>/\t/g')
    message=$(
        echo "{"
        echo "$to_parse" | awk '{ print "\"Channel\": "$1","
                                                            print "\"LockStatus\": \""$2"\","
                                                            print "\"Modulation_ProfileID\": \""$3"\","
                                                            print "\"ChannelID\": "$4","
                                                            print "\"Frequency\": "$5","
                                                            print "\"Power\": "$6","
                                                            print "\"SNR_MER\" :"$7","
                                                            print "\"ActiveSubcarrier_NumberRange\": \""$8"\","
                                                            print "\"Unerrored\" :"$9","
                                                            print "\"Corrected\" :"$10","
                                                            print "\"Uncorrectable\" :"$11 }'
        echo "}"
    )
    $mqtt_pub_exe -h "$mqtt_broker" -u "$mqtt_username" -P "$mqtt_password" -t "${mqtt_topic}/ofdm_downstream/${counter}" -m "$message"
done

# Parse out the OFDM upstream HTML table into JSON and publish
#echo "$upstream_rows" | sed 's/<th[^>]*>[^<]*<\/th><\/tr>//g;s/^<td>//g;s/<\/td>$//g;s/<\/td><td[^>]*>/\t/g'
counter=0
echo "$ofdm_upstream_rows" | tail -n +3 | while read -r line; do
    counter=$(($counter+1))
    #echo "${mqtt_topic}/upstream/$counter"
    to_parse=$(echo "$line" | sed 's/<th[^>]*>[^<]*<\/th><\/tr>//g;s/^<td>//g;s/<\/td>$//g;s/<\/td><td[^>]*>/\t/g')
    message=$(
        echo "{"
        echo "$to_parse" | awk '{ print "\"Channel\": "$1","
                                                            print "\"LockStatus\": \""$2"\","
                                                            print "\"Modulation_ProfileID\": \""$3"\","
                                                            print "\"ChannelID\": "$4","
                                                            print "\"Frequency\": "$5","
                                                            print "\"Power\": "$6 }'
        echo "}"
    )
    $mqtt_pub_exe -h "$mqtt_broker" -u "$mqtt_username" -P "$mqtt_password" -t "${mqtt_topic}/ofdm_upstream/${counter}" -m "$message"
done

# Publish the system up time from the modem
message="{ \"SystemUpTime\": \"$system_up_time\" }"
$mqtt_pub_exe -h "$mqtt_broker" -u "$mqtt_username" -P "$mqtt_password" -t "${mqtt_topic}/system_up_time" -m "$message"

#echo ""

putty: image

MQTT Explorer: image

image

DeFlanko commented 9 months ago

Let me know if it's helpful or not to have the htm from "https://192.168.100.1/DocsisStatus.htm"

DeFlanko commented 9 months ago

Ive spent most of the day troubleshooting this and im no where closer. image

image

#!/bin/bash

###########################
# USER DEFINED PARAMETERS #
###########################
#sprinkle this in spots to help troubleshoot where in the code the cod eis having an issue
#echo "Line: ${LINENO}"

# Default Modem Paramters
modem_username="admin"
modem_password="---REMOVED---"
modem_ip="192.168.100.1"

# Modem URL paths
DOCSIS_URL="https://192.168.100.1"
DOCSIS_STATUS_PAGE="DocsisStatus.htm"
DOCSIS_LOGS="eventLog.htm"

# Settings for MQTT mqtt_broker to publish stats
mqtt_broker="172.10.1.12"
mqtt_username="homeassistant"
mqtt_password="---REMOVED---"
mqtt_topic="homeassistant/sensor/modemsignals"

# HomeAssistant doesn't expose this to the container so we have to hack it up
# Comment these out for a "normal" host that knows where mosquitto_pub is on its own
export LD_LIBRARY_PATH='/config/bin/mosquitto_deps/lib'
mqtt_pub_exe="/config/bin/mosquitto_deps/mosquitto_pub"

# Uncomment tthis for a "normal" host that knows where mosquitto_pub is on its own
#mqtt_pub_exe="mosquitto_pub"

# Dynamic Cookie file path
cookie_path="$0.cookie"
#testing
#cookie_path='/config/netgear_modem_signal_scraper/netgear_signal_dump.cookie'

#####################################
# Prep functions to interface modem #
#####################################

# This function publishes login status helpful for debugging

function MQTTStatus () {
    #echo "Modem login: $1"
    # Publish MQTT to announce status
    if [ "$2" != "" ]; then
        message="{ \"login\": \"$1\", \"detail\": \"$2\" }"
    else
        message="{ \"login\": \"$1\" }"
    fi
    $mqtt_pub_exe -h "$mqtt_broker" -u "$mqtt_username" -P "$mqtt_password" -t "${mqtt_topic}/login" -m "$message" || echo "MQTT-Pub Error!"
}

# Fetch the page and then print the result
webToken=""
realSessionId=""
lastPageFetch=""

#echo "Line: ${LINENO}"

function pageDive () {
  toFetch="$1"
  referPage="$2"
  depth=$(( $depth + 1 ))
  lastPageFetch="$toFetch"

  if [[ "$depth" -ge 5 ]]; then
    MQTTStatus "max_redirect_exceeded"
  else

      # Fetch the page
      if [ "$referPage" != "" ]; then
        referURL="${DOCSIS_URL}"
      fi

        # Decide which request to make (if page requires post data)
        if [ "$toFetch" == "/DocsisStatus.htm" ]; then
            data=$(curl "$referURL" -k --connect-timeout 20 -v -L -c "$cookie_path" -b "$cookie_path" -F "loginUsername=${modem_username}" -F "loginPassword=${modem_password}" "${DOCSIS_URL}${toFetch}" 2>&1) # -e -i "${DOCSIS_URL}${toFetch}"  --data-urlencode "loginUsername=$modem_username" --data-urlencode "loginPassword=$modem_password"
            exitCode=$?
        fi

        # if [ "$toFetch" == "/goform/GenieLogin" ]; then
        #   data=$(curl --connect-timeout 20 -v -s -e "$referURL" -c "$cookie_path" -b "$cookie_path" -X POST -H 'Content-Type: application/x-www-form-urlencoded' --data-urlencode "loginUsername=$modem_username" --data-urlencode "loginPassword=$modem_password" --data "login=1" --data "webToken=$webToken" "${DOCSIS_URL}${toFetch}"  2>&1)
        #   exitCode=$?
        # elif [ "$toFetch" == "/goform/MultiLogin" ]; then
        #   data=$(curl --connect-timeout 20 -v -s -e "$referURL" -c "$cookie_path" -b "$cookie_path" -X POST -H 'Content-Type: application/x-www-form-urlencoded' --data "yes=yes" --data "Act=yes" --data "RetailSessionId=$retailSessionId" --data "webToken=$webToken" "${DOCSIS_URL}${toFetch}"  2>&1)
        #   exitCode=$?
        # else
        #   #data=$(curl --connect-timeout 10 -v -s -e "$referURL" -b "$cookie_path" -c "$cookie_path" "${DOCSIS_URL}${toFetch}" 2>&1)
        #   data=$(curl "$referURL" -k --connect-timeout 20 -v -L -c "$cookie_path" -b "$cookie_path" -F "loginUsername=${modem_username}" -F "loginPassword=${modem_password}" 2>&1) # -e -i "${DOCSIS_URL}${toFetch}"  --data-urlencode "loginUsername=$modem_username" --data-urlencode "loginPassword=$modem_password"
        #   exitCode=$?
        # fi

      # Check for timeout and decide what to do next
      if [ "$exitCode" == 28 ]; then 
        MQTTStatus "failed_timeout" "${toFetch}"
        exit 11
      fi

      # Get redirect location if any
      redirectPage=$(echo "$data" | egrep 'window.top.location|< Location: ' | awk '{print $3}' | sed 's/[";]//g')

      # Pull out hidden fields if applicable
      webToken=$(echo "$data" | egrep -o 'name="webToken" value=[0-9]+' | egrep -o '[0-9]+')
      retailSessionId=$(echo "$data" | egrep -o 'name=\"RetailSessionId\" value=\"[^\"]*\"' | awk -F "\"" '{print $4}')

      # Decide if we need to submit a form and redirect
      #MQTTStatus "toFetch=$toFetch"
      if [ "$toFetch" == "/GenieLogin.asp" ]; then
        redirectPage="/goform/GenieLogin"
      elif [ "$toFetch" == "/MultiLogin.asp" ]; then
        redirectPage="/goform/MultiLogin"
      else
        redirectPage="/DocsisStatus.htm"
        # MQTTStatus "Line ${LINENO}"
      fi

      # Drill down if applicable
      if [ "$redirectPage" == "" ]; then
        echo "$data"
      else
        MQTTStatus "redirect" "${redirectPage}"
        pageDive "$redirectPage" "$toFetch"
      fi
  fi
  depth=$(( $depth - 1 ))
}

# This function gets a dynamic session-cookie from the modem
function getSession () {

    # Perform the login, or verify we are already logged in
    pageDive > /dev/null 

}

# This function fetches the HTML status page from the modem for parsing
function getResult () {
# Finally, we can request the page

    result=$(pageDive '/DocsisStatus.htm' 'GenieIndex.asp')

    # If we were given GenieIndex instead of DocsisStatus, that means it went thru a redirect
    # so we will try exactly one more time.
    if [ "$(echo "$result" | grep -c '> GET /GenieIndex.asp HTTP/1.1')" == "1" ] ; then
        result=$(pageDive '/DocsisStatus.htm' 'GenieIndex.asp')
    fi

}

#############################
# Log in and fetch the data #
#############################

# Get the result from the modem
#getSession;
getResult;

# See if we were successful
if [ "$(echo "$result" | grep -c 'Downstream Bonded Channels')" == "0" ] ; then
    MQTTStatus "failed_retrying"

#   # If we failed (got a login prompt) try once more for new token
    rm "$cookie_path"
    pageDive '/MultiLogin.asp' > /dev/null
    pageDive '/Logout.asp' > /dev/null
    pageDive '/Logout.asp' > /dev/null
    rm "$cookie_path"

    getSession;
    getResult;
fi

echo "------------------------- TEST PARAMS on line: ${LINENO}-------------------------"
echo "Base: ${DOCSIS_URL}"          #https://192.168.100.1
echo "toFetch: ${toFetch}"
echo "Cookie: ${cookie_path}"   #/config/netgear_modem_signal_scraper/netgear_signal_dump.cookie
echo "referURL: ${referURL}"    #https://192.168.100.1/DocsisStatus.htm
echo "redirectPage: ${redirectPage}"
echo "referpage: ${referPage}"
echo "getSession: ${getSession}"
echo "getresult: ${getResult}"
echo "Webtoken: ${webToken}"
echo "realSessionId: ${realSessionId}"
echo "lastPageFetch: ${lastPageFetch}"
echo "---------------------------------------------------------------------------------"
echo "data: ${data}"    #data: * WARNING: failed to open cookie file "/config/netgear_modem_signal_scraper/netgear_signal_dump.cookie"
echo "---------------------------------------------------------------------------------"
#echo "Line: ${LINENO}"

# See if we were successful
if [ "$(echo "$result" | grep -c 'Downstream Bonded Channels')" == "0" ] ; then
    # At this point, if we weren't successful, we give up
    MQTTStatus "failed to get Downstream Bonded Channels"
    echo "Line: ${LINENO}"
    exit 21
else
    MQTTStatus "success"
    echo "Line: ${LINENO}"
    #echo "$result"
fi

echo "Line: ${LINENO}"

# Log out afterward
#pageDive '/Logout.asp' > /dev/null

####################
# Parse the result #
####################

#echo "Raw:"
#echo -e "$result"

#echo "$result" | tr '\n' ' ' | sed 's/\t//g;s/ //g;s/dBmV//g;s/dB//g' | awk -F "<tableclass=['\"]simpleTable['\"]>|</table>" '{print "\nStartup:\n" $2 "\n\nDown\n" $4 "\n\nUp\n" $6 "\n" }'
#startup_status=$(echo "$result" | tr '\n' ' ' | sed 's/\t//g;s/ //g;s/dBmV//g;s/dB//g;s/Hz//g;s/<[/]\?strong>//g;s/<![^>]*>//g;s/<[/]\?[bui]>//g' | awk -F "tableid=\"startup_procedure_table\"" '{print $2}'  )
startup_status=$(echo "$result" | tr '\n\r' ' ' | sed 's/\t//g;s/ //g;s/<!--Hiddenconfigfilefield-->//g' | awk -F "tableid=\"startup_procedure_table\"" '{print $2}' | awk -F "</table>" '{print $1}')
downstream_status=$(echo "$result" | tr '\n' ' ' | sed 's/\t//g;s/ //g;s/dBmV//g;s/dB//g;s/Hz//g;s/<[/]\?strong>//g;s/<![^>]*>//g' | awk -F "<[/]?tabindex=-1>" '{print $5}' | awk -F "</table>" '{print $1}')
upstream_status=$(echo "$result" | tr '\n' ' ' | sed 's/\t//g;s/ //g;s/dBmV//g;s/dB//g;s/Hz//g;s/<[/]\?strong>//g;s/<![^>]*>//g' | awk -F "<[/]?tabindex=-1>" '{print $7}' | awk -F "</table>" '{print $1}')
ofdm_downstream_status=$(echo "$result" | tr '\n' ' ' | sed 's/\t//g;s/ //g;s/dBmV//g;s/dB//g;s/Hz//g;s/<[/]\?strong>//g;s/<![^>]*>//g' | awk -F "<[/]?tabindex=-1>" '{print $9}' | awk -F "</table>" '{print $1}')
ofdm_upstream_status=$(echo "$result" | tr '\n' ' ' | sed 's/\t//g;s/ //g;s/dBmV//g;s/dB//g;s/Hz//g;s/<[/]\?strong>//g;s/<![^>]*>//g' | awk -F "<[/]?tabindex=-1>" '{print $11}' | awk -F "</table>" '{print $1}')
system_up_time=$(echo "$result" | grep "SystemUpTime" | awk -F "</b>|</font>" '{print $2}')

# Break out by line
startup_rows=$(echo "$startup_status" | sed 's/^<tr>//g;s/<\/tr>$//g;s/<\/tr><tr[^>]*>/\n/g')
downstream_rows=$(echo "$downstream_status" | sed 's/^<tr>//g;s/<\/tr>$//g;s/<\/tr><tr[^>]*>/\n/g')
upstream_rows=$(echo "$upstream_status" | sed 's/^<tr>//g;s/<\/tr>$//g;s/<\/tr><tr[^>]*>/\n/g')
ofdm_downstream_rows=$(echo "$ofdm_downstream_status" | sed 's/^<tr>//g;s/<\/tr>$//g;s/<\/tr><tr[^>]*>/\n/g')
ofdm_upstream_rows=$(echo "$ofdm_upstream_status" | sed 's/^<tr>//g;s/<\/tr>$//g;s/<\/tr><tr[^>]*>/\n/g')
# Note: system_up_time is a single value and does not need additional parsing

# Break out columns

# Parse out the startup status HTML table into JSON and publish
#echo "$startup_rows"
#echo "$startup_rows" | sed 's/<th[^>]*>[^<]*<\/th>//g;s/^<td[^>]*>//g;s/<\/td>$//g;s/<\/td><td[^>]*>/\t/g' | grep -v "^$"
# Helper function to more easily build JSON per field
echo "Line: ${LINENO}"
function pubStartupStatusValue () {
    # Break out field information
    procedure_name="$1"
    procedure_status="$2"
    procedure_comment="$3"
    # Build the message payload
    message=""
    # If exists, insert stattus
    if [ "$procedure_status" != "" ]; then
        if [[ "$procedure_status" =~ ^[0-9]+$ ]]; then
            message="${message} \"status\": $procedure_status"
        else
            message="${message} \"status\": \"$procedure_status\""
        fi
    fi
    # If exists, insert comment
    if [ "$procedure_comment" != "" ]; then
        # If message is not empty, insert separator comma
        if [ "$message" != "" ]; then
            message="${message}, "
        fi
        message="${message} \"comment\": \"$procedure_comment\""
    fi
    message="{ ${message} }"
    $mqtt_pub_exe -h "$mqtt_broker" -u "$mqtt_username" -P "$mqtt_password" -t "${mqtt_topic}/startup_procedure/${procedure_name}" -m "$message"
}
echo "$startup_rows" | grep -v "^$" | tail -n +2 | while read -r line; do
    to_parse=$(echo "$line" | sed 's/<th[^>]*>[^<]*<\/th>//g;s/^<td[^>]*>//g;s/<\/td>$//g;s/<\/td><td[^>]*>/\t/g')
    procedure_name=$(echo $to_parse | awk '{print $1}')
    procedure_status=$(echo $to_parse | awk '{print $2}')
    procedure_comment=$(echo $to_parse | awk '{print $3}')
    pubStartupStatusValue "$procedure_name" "$procedure_status" "$procedure_comment"
done

echo "line 249"

# Parse out the downstream HTML table into JSON and publish
#echo "$downstream_rows" | sed 's/<th[^>]*>[^<]*<\/th><\/tr>//g;s/^<td>//g;s/<\/td>$//g;s/<\/td><td[^>]*>/\t/g'
counter=0
echo "$downstream_rows" | tail -n +3 | while read -r line; do
    counter=$(($counter+1))
    #echo "${mqtt_topic}/downstream/$counter"
    to_parse=$(echo "$line" | sed 's/<th[^>]*>[^<]*<\/th><\/tr>//g;s/^<td>//g;s/<\/td>$//g;s/<\/td><td[^>]*>/\t/g')
    message=$(
        echo "{"
        echo "$to_parse" | awk '{ print "\"Channel\": "$1","
                                                            print "\"LockStatus\": \""$2"\","
                                                            print "\"Modulation\": \""$3"\","
                                                            print "\"ChannelID\": "$4","
                                                            print "\"Frequency\": "$5","
                                                            print "\"Power\": "$6","
                                                            print "\"SNR_MER\" :"$7","
                                                            print "\"Unerrored\" :"$8","
                                                            print "\"Corrected\" :"$9","
                                                            print "\"Uncorrectable\" :"$10 }'
        echo "}"
    )
    $mqtt_pub_exe -h "$mqtt_broker" -u "$mqtt_username" -P "$mqtt_password" -t "${mqtt_topic}/downstream/${counter}" -m "$message"
done

# Parse out the upstream HTML table into JSON and publish
#echo "$upstream_rows" | sed 's/<th[^>]*>[^<]*<\/th><\/tr>//g;s/^<td>//g;s/<\/td>$//g;s/<\/td><td[^>]*>/\t/g'
counter=0
echo "$upstream_rows" | tail -n +3 | while read -r line; do
    counter=$(($counter+1))
    #echo "${mqtt_topic}/upstream/$counter"
    to_parse=$(echo "$line" | sed 's/<th[^>]*>[^<]*<\/th><\/tr>//g;s/^<td>//g;s/<\/td>$//g;s/<\/td><td[^>]*>/\t/g')
    message=$(
        echo "{"
        echo "$to_parse" | awk '{ print "\"Channel\": "$1","
                                                            print "\"LockStatus\": \""$2"\","
                                                            print "\"Modulation\": \""$3"\","
                                                            print "\"ChannelID\": "$4","
                                                            print "\"Frequency\": "$5","
                                                            print "\"Power\": "$6 }'
        echo "}"
    )
    $mqtt_pub_exe -h "$mqtt_broker" -u "$mqtt_username" -P "$mqtt_password" -t "${mqtt_topic}/upstream/${counter}" -m "$message"
done

# Parse out the OFDM downstream HTML table into JSON and publish
counter=0
echo "$ofdm_downstream_rows" | tail -n +3 | while read -r line; do
    counter=$(($counter+1))
    to_parse=$(echo "$line" | sed 's/<th[^>]*>[^<]*<\/th><\/tr>//g;s/^<td>//g;s/<\/td>$//g;s/<\/td><td[^>]*>/\t/g')
    message=$(
        echo "{"
        echo "$to_parse" | awk '{ print "\"Channel\": "$1","
                                                            print "\"LockStatus\": \""$2"\","
                                                            print "\"Modulation_ProfileID\": \""$3"\","
                                                            print "\"ChannelID\": "$4","
                                                            print "\"Frequency\": "$5","
                                                            print "\"Power\": "$6","
                                                            print "\"SNR_MER\" :"$7","
                                                            print "\"ActiveSubcarrier_NumberRange\": \""$8"\","
                                                            print "\"Unerrored\" :"$9","
                                                            print "\"Corrected\" :"$10","
                                                            print "\"Uncorrectable\" :"$11 }'
        echo "}"
    )
    $mqtt_pub_exe -h "$mqtt_broker" -u "$mqtt_username" -P "$mqtt_password" -t "${mqtt_topic}/ofdm_downstream/${counter}" -m "$message"
done

# Parse out the OFDM upstream HTML table into JSON and publish
#echo "$upstream_rows" | sed 's/<th[^>]*>[^<]*<\/th><\/tr>//g;s/^<td>//g;s/<\/td>$//g;s/<\/td><td[^>]*>/\t/g'
counter=0
echo "$ofdm_upstream_rows" | tail -n +3 | while read -r line; do
    counter=$(($counter+1))
    #echo "${mqtt_topic}/upstream/$counter"
    to_parse=$(echo "$line" | sed 's/<th[^>]*>[^<]*<\/th><\/tr>//g;s/^<td>//g;s/<\/td>$//g;s/<\/td><td[^>]*>/\t/g')
    message=$(
        echo "{"
        echo "$to_parse" | awk '{ print "\"Channel\": "$1","
                                                            print "\"LockStatus\": \""$2"\","
                                                            print "\"Modulation_ProfileID\": \""$3"\","
                                                            print "\"ChannelID\": "$4","
                                                            print "\"Frequency\": "$5","
                                                            print "\"Power\": "$6 }'
        echo "}"
    )
    $mqtt_pub_exe -h "$mqtt_broker" -u "$mqtt_username" -P "$mqtt_password" -t "${mqtt_topic}/ofdm_upstream/${counter}" -m "$message"
done

# Publish the system up time from the modem
message="{ \"SystemUpTime\": \"$system_up_time\" }"
$mqtt_pub_exe -h "$mqtt_broker" -u "$mqtt_username" -P "$mqtt_password" -t "${mqtt_topic}/system_up_time" -m "$message"

#echo ""

Here is what i do know...

  1. /DocsisStatus.htm is where i wanted to end up to start the scrape
  2. renaming LoginStatus to MQTTStatus was way more helpful for me
  3. echo "Line: ${LINENO}" is fantastic
  4. MQTTStatus "toFetch=$toFetch" did help identify that it was trying to go to "/DocsisStatus.asp" but when i tried to add that as a new elif it didn't liek it so i just made the else default to /DocsisStatus.htm
  5. i think the $data curl might still need tweaking(?)
mmiller7 commented 8 months ago

At the moment its been long enough I can't remember exactly what I wrote here, and have a lot going on so I can't try and mentally step thru it. I can try and revisit this if I have more time later this year perhaps.

Sorry that's not super helpful at the moment for you :/

DeFlanko commented 8 months ago

No Worries miller. Take all the time you need.

On Sun, Oct 22, 2023 at 1:02 PM mmiller7 @.***> wrote:

At the moment its been long enough I can't remember exactly what I wrote here, and have a lot going on so I can't try and mentally step thru it. I can try and revisit this if I have more time later this year perhaps.

Sorry that's not super helpful at the moment for you :/

— Reply to this email directly, view it on GitHub https://github.com/mmiller7/Netgear_modem_scrape/issues/1#issuecomment-1774185949, or unsubscribe https://github.com/notifications/unsubscribe-auth/ABH52MJTKTECIBQZUHBZDK3YAV3WTAVCNFSM5SG5VDJ2U5DIOJSWCZC7NNSXTN2JONZXKZKDN5WW2ZLOOQ5TCNZXGQYTQNJZGQ4Q . You are receiving this because you modified the open/close state.Message ID: @.***>

-- Thanks, James