/***************************************************************************/
/* EchoStat.spot (C) Phil O'Malley 1994, Fidonet: 2:250/107.96             */
/*                                                                         */
/* $VER: EchoStat [15.02.94]                                               */
/***************************************************************************/
/*                                                                         */
/* Desc : A script which analyzes the messages held in an area and then    */
/*        produces traffic report.                                         */
/*                                                                         */
/***************************************************************************/
/*                                                                         */
/* Usage: (1) Invoke from the CLI with Spot as a background task.          */
/*                                                                         */
/*              ie. rx >nil: rexx:EchoStat.spot                            */
/*                                                                         */
/*        (2) Or specify the desired syntax from the Spot ARexx menu.      */
/*                                                                         */
/***************************************************************************/
/*                                                                         */
/* Ctrl : By CLI parameters. These are:                                    */
/*                                                                         */
/*   NONODE      = Don't append the table concerning non-point origins.    */
/*   NOSUBJECT   = Don't append the table concerning echomail subjects.    */
/*   NOTRAFFIC   = Don't append the table concerning echomail traffic.     */
/*                                                                         */
/*   NOFLAG      = Don't display the flags in the output tables.           */
/*   NOPOST      = Don't post the final output into the current area.      */
/*                                                                         */
/*   NTC <%>     = Alter the wasting bandwidth flag threshold to <%>.      */
/*   XSQUOTE <%> = Alter the excessive quoting flag threshold to <%>.      */
/*   TABLE <len> = Alter the table length (0 = no limits).                 */
/*                                                                         */
/*   OUT <file>  = Specify an output path for the ASCII report.            */
/*                                                                         */
/***************************************************************************/

address spot 
options results

/***************************************************************************/
/* Check for, and install if necessary, any libraries we need.             */
/***************************************************************************/

if ~show(l, "traplist.library") then do
  if ~addlib("traplist.library", 0, -30) then do
    say "Please install the traplist.library in your libs: directory"
  end
end

/***************************************************************************/
/* Define global variables                                                 */
/***************************************************************************/

version = 'EchoStat 1.1'

/* Logical assigns */

true    = 1
false   = 0

blank   = ''
newline = '0a'x

/* File names. */

temp_file   = 'T:EchoStat.tmp'
output_file = 'T:EchoStat.tmp'

/* Defaults. Can be overriden from the CLI. */

xsquoting_level  = 57.5
newtopic_content = 37.5
table_size       = 10

/***************************************************************************/
/* Grab the command line and check to see if (crudely) valid.              */
/***************************************************************************/

parse arg command_line

if command_line = "?" then do
  say "Usage: rx EchoStat.spot [NOFLAG] [NONODE] [NOPOST] [NOSUBJECT] [NOTRAFFIC] [NTC %] [OUT <file>] [TABLE <length>] [XSQUOTE %]" ; exit
end

/***************************************************************************/
/* Main program. All procedures are called from here.                      */
/***************************************************************************/

/* Make sure that any other requesters are closed. */

'progressclose 1'

/* Parse the CLI. */

call parsecli()

/* Review the messagebase. */

call parse_statistics()

/* Now sort the report. All output from the procedures is sent back in
   the form of a variable which is organised into the report later. */

if notraffic ~= true then do
  report.1 = produce_traffic_statistics(table_size)
end

if nosubject ~= true then do
  report.2 = produce_subject_statistics(table_size)
end

if nonode ~= true then do
  report.3 = produce_node_statistics(table_size)
end

/* Sort out the outputfile. */

call produce_report()

/* Now, if not overridden from the CLI, post the output as a message in the
   current area. */

if nopost ~= true then do
  'write TO "All" SUBJECT "Echo Stats: 'date(e)'" FILE "'output_file'" NOEDIT NOGUI REFLOW=OFF'
end

exit

/***************************************************************************/
/* ParseCLI(). Parse the command line.                                     */
/***************************************************************************/

parsecli: procedure expose command_line noflag nonode nopost nosubject nosubject newtopic_content output_file table_size true xsquoting_level 

  /* Each element is separated by a space, thus we can parse them
     with as if it was a simple sentence. */

  do i = 1 to words(command_line)

    if upper(word(command_line,i)) = "NOFLAG" then do
      noflag = true
    end

    if upper(word(command_line,i)) = "NONODE" then do
      nonode = true
    end

    if upper(word(command_line,i)) = "NOPOST" then do
      nopost = true 
    end

    if upper(word(command_line,i)) = "NOSUBJECT" then do
      nosubject = true
    end

    if upper(word(command_line,i)) = "NOTRAFFIC" then do
      notraffic = true 
    end

    if upper(word(command_line,i)) = "NTC" then do
      newtopic_content = word(command_line,(i+1)) ; i = i +1
    end

    if upper(word(command_line,i)) = "OUT" then do
      output_file = word(command_line,(i+1)) ; i = i +1
    end

    if upper(word(command_line,i)) = "TABLE" then do
      table_size = word(command_line,(i+1)) ; i = i +1
    end

    if upper(word(command_line,i)) = "XSQUOTE" then do
      xsquoting_level = word(command_line,(i+1)) ; i = i +1
    end

  end

return

/***************************************************************************/
/* Produce_Report(). Create/append the output file.                        */
/***************************************************************************/

produce_report: procedure expose report. newline output_file version

  /* This holds the organisational data for the report file. The report.x
     variables are spooled into the file in the order designated allowing
     greater flexibility than if just dumped into a log file. */

  report_format = "1 2 3"

  /* Find the number of valid reports. */

  reports_remaining = 0

  do i = 1 to words(report_format)
    number = word(report_format,i)

    if report.number ~= upper("report."number) then do
      reports_remaining = reports_remaining +1
    end 
  end

  if reports_remaining ~= 0 then do

    /* Grab the area name and number of messages. */

    'getareaname' ; area_name = result
    'getnummsgs' ; number_messages = result

    call open(report,output_file,'w')
    call writeln(report,'Report for 'area_name' ('number_messages') on 'date(e)':')

    /* And now we parse the report format to create the report. */

    do i = 1 to words(report_format)
      number = word(report_format,i)

      if report.number ~= upper("report."number) then do
        call writeln(report,report.number) ; reports_remaining = reports_remaining -1
      end 
    end
 
    /* Close the report. */

    call writech(report,newline'Created by 'version' (C)1994 Phil O''Malley (FIDONET#2:250/107.96)')
    call close(report)

  end

return

/* ----------------------------------------------------------------------- */

/***************************************************************************/
/* Parse_Statistics(). Grab all statistics from the current area.          */
/***************************************************************************/

parse_statistics: procedure expose true false temp_file subjecttotal. nodetotal. address. nodelist. subjectlist. userlist. receiver. writer. writer_q. writer_nq. subject_q. subject_nq. version

  /* Grab the current messgae and then mark all unread messages so that we
     can restore the area to how me found it later.  */

  'getmessagenum'
   current_message =  result

  'messagelist' ; 'includeflag unread'
  'gotomessage 1' ; 'messages'

  /* Grab the preq values and pop open the progress requester. */

  'progressopen title "'version'" prompt "Assessing Echomail Information..."'
   preq = result ; 'getnummsgs' ; ptotal = result ; pcurrent = 1 

  /* The userlist. stem variable is used to store firstly the total number
     of users (both as writers and receivers) in the area in branch .0 with
     subsequent branches holding the names themselves. All other variables
     are branched from these names which is the quickest option. */

  userlist.0 = 0 ; subjectlist.0 = 0 ; nodelist.0 = 0

  /* Scan all messages in the area... */

  do loop = 1 to ptotal

  /* Obtain the user name, and address. As with all entries, first we
     check to see if one is already present for the person concerned, and
     if not we make on and initialise all the variables. */

    'getfrom' ; getfrom_result = result
    'getfromaddress' ; getfromaddress_result = result

    if writer.getfrom_result = 'WRITER.'getfrom_result & receiver.getfrom_result = 'RECEIVER.'getfrom_result then
      do
        userlist.0 = userlist.0 + 1 ; usertotal = userlist.0
        userlist.usertotal = getfrom_result

        /* For each user entry, we must initialise all the variables which
           could be needed. One of the more dubious features of ARexx. */

        writer.getfrom_result = 1 ; writer_nq.getfrom_result = 0 ; writer_q.getfrom_result = 0
        receiver.getfrom_result = 0 ; address.getfrom_result = getfromaddress_result
      end
    else
      do  
        writer.getfrom_result = writer.getfrom_result + 1
        address.getfrom_result = getfromaddress_result
      end

    /* Find out where the message is going. */

    'getto' ; getto_result = result

    if receiver.getto_result = 'RECEIVER.'getto_result & writer.getto_result = 'WRITER.'getto_result then
      do
        userlist.0 = userlist.0 + 1 ; usertotal = userlist.0
        userlist.usertotal = getto_result

        receiver.getto_result = 1 ; address.getto_result = "?"
        writer.getto_result = 0 ; writer_nq.getto_result = 0 ; writer_q.getto_result = 0
      end
    else 
      receiver.getto_result = receiver.getto_result + 1

    /* Now grab the subject information and append it to its own series
       of variable stems. Quoting stats will also be included so we can run
       and waste-of-bandwidth check later on. */

    'getsubject' ; getsubject_result = result 

    /* Strip out RE:s */

    if index(upper(getsubject_result),"RE: ") ~= 0 then do
      getsubject_result = right(getsubject_result,(length(getsubject_result)-4))
    end

    if subjecttotal.getsubject_result = 'SUBJECTTOTAL.'getsubject_result then
      do
        subjectlist.0 = subjectlist.0 + 1 ; subjecttotal = subjectlist.0
        subjectlist.subjecttotal = getsubject_result

        subjecttotal.getsubject_result = 1 ; subject_nq.getsubject_result = 0 ; subject_q.getsubject_result = 0
      end
    else 
      subjecttotal.getsubject_result = subjecttotal.getsubject_result + 1

    /* Now things will slow down. Unfortunately, we have to access all
       messages in the message window which adds to the time taken to dump
       out the message as an ASCII file for analysis purposes. */

    'saveascii 'temp_file' overwrite noheader nokludges notearline noorigin'

    call open(file,temp_file,"r")

      do until eof(file) = 1
        line = readln(file)

        if index(line,">")>1 & index(line,">")<6 then
          do
            writer_q.getfrom_result = writer_q.getfrom_result + length(line)
            subject_q.getsubject_result = subject_q.getsubject_result + length(line)
          end
        else
          if line ~= "" then do 
            writer_nq.getfrom_result = writer_nq.getfrom_result + length(line)
            subject_nq.getsubject_result = subject_nq.getsubject_result + length(line)
          end

      end

    call close(file)    

    /* Update the progress requester... */

    'progressupdate 'preq' current 'pcurrent' total 'ptotal

    if rc = 5 then do
      'progressclose 'preq ; exit
    end

    /* We must see if we are at the last message in the area as if we try
       to grab the next message then the script fails with an error. */

    if loop ~= ptotal then do
      pcurrent = pcurrent +1 ; 'nextmessage'
    end

  end

  'progressclose 'preq

  /* Restore any unread flags. */

  'messagelist' ; 'setflags unread' ; 'excludeflag all'

return

/***************************************************************************/
/* Produce_Traffic_Statistics(). Total users, mail from/to, quoting %      */
/***************************************************************************/

produce_traffic_statistics: procedure expose true blank newline noflag address. receiver. userlist. writer. writer_nq. writer_q. version xsquoting_level

  parse arg table_limit ; if table_limit = "" then table_limit = 0 ; total_entries = 0 ; total_writers = 0 ; total_receivers = 0

  do entry = 1 to userlist.0

    username = userlist.entry

    select
      when writer_q.username = 0 then writer_qs.username = 0
      when writer_nq.username = 0 then writer_qs.username = 100

      otherwise writer_qs.username = (writer_q.username/(writer_nq.username+writer_q.username))*100
    end
  end

  /* Write the message header. */

  if userlist.0 ~= 0 then do

    /* Initialise the report. The header will be appended later when we've
       found how many writers and recievers there were. */

    report = blank

    /* Sort according to amount of mail posted. */

    'progressopen title "'version'" prompt "Sorting Echomail Traffic..."'
     preq = result ; current_entry = 1

    do loop = 1 to userlist.0
      writer = 0 ; highest = 0

        do u = 1 to userlist.0
          username = userlist.u

          if writer.username ~< highest & writer_used.username ~= "Null" then do
            writer = username ; highest = writer.username
          end
        end

      /* Update the total number of senders and receivers. */

      if writer.writer ~= 0 then total_writers = total_writers +1
      if receiver.writer = ~0 then total_receivers = total_receivers +1

      /* Work out any flags to be displayed. Notice that the flags are
         entered in order of dominance as only one is displayed. */

      select
        when noflag = true then flag = " "
        when address.writer = "?" then flag = " "
        when display_traplist_info(left((address.writer),((index((address.writer),".",1))-1))) = "?" then flag = "#"
        when writer_qs.writer > xsquoting_level then flag = "!"

        otherwise flag = " "
      end

      /* Keep tabs on any flags used so that we can append a key to the
         bottom of the table. */

      if flag = "!" then flag_! = true
      if flag = "#" then flag_# = true

      if (loop > table_limit) & table_limit ~= 0 & flag = " " then
        do 
          /* Do nothing */
        end
      else
        if (loop > table_limit) & table_limit ~= 0 then
          do 
            report = report||newline"  | "||right(loop,3," ")||". "||flag||" "||left(writer||" ",22,".")||" "||left(address.writer||" ",14,".")||right(" "||writer.writer||" ",7,".")||right(" "||receiver.writer||" ",7,".")||right(left((" "||writer_qs.writer||".0"),((index((" "||writer_qs.writer||".0"),".",1))+1)),8,".")||"%"||"  |"
            extra_entries = true
          end
        else
          report = report||newline"  | "||right(loop,3," ")||". "||flag||" "||left(writer||" ",22,".")||" "||left(address.writer||" ",14,".")||right(" "||writer.writer||" ",7,".")||right(" "||receiver.writer||" ",7,".")||right(left((" "||writer_qs.writer||".0"),((index((" "||writer_qs.writer||".0"),".",1))+1)),8,".")||"%"||"  |"

      /* This "Null" business is for convience during sorting. Notice that
         it uses it's own variable and so is completely forgotten about when
         we leave the procedure. */

      writer_used.writer = "Null"

      /* Check to see if we've reached the desired table size. */

      if (loop = table_limit) & table_limit ~= 0 then do
        report = report||newline'  +----------------------------------------------------------------------+'
        table_limit_seperator = true
      end

      /* Update the progress requester. */

      'progressupdate 'preq' current 'loop' total 'userlist.0

       if rc = 5 then do
         'progressclose 'preq ; exit
       end

    end

    if table_limit_seperator ~= true | extra_entries = true then do
      report = report||newline'  +----------------------------------------------------------------------+'
    end

    if flag_! = true | flag_# = true then do

      footnote = "NB."

      if flag_! = true then do
        report = report||newline'  |  'footnote' ! = Quoting level is considered excessive.                      |' ; footnote = "   "
      end

      if flag_# = true then do
        report = report||newline'  |  'footnote' # = Address is not present in the current nodelist.             |' ; footnote = "   "
      end

      report = report||newline'  +----------------------------------------------------------------------+'

    end

    /* Create the header, taking into account the total number of people in
       in echo and the total number of writers and receivers. */

    header_title = '       | Echomail Traffic (T:'userlist.0', W:'total_writers', R:'total_receivers') |'    

    header =  blank||newline'       +'||right(blank,(length(header_title)-9),"-")||'+'
    header = header||newline||header_title
    header = header||newline'  +----+'||right(blank,(length(header_title)-9),"-")||'+'||right(blank,(73-length(header_title)),"-")||'+'
    header = header||newline'  |  No.   Name                   Address         From     To    Quoted  |'
    header = header||newline'  +----------------------------------------------------------------------+'

    'progressclose 'preq

  end

return header||report

/***************************************************************************/
/* Procduce_Subject_Statistics(). Subject popularity and debate.           */
/***************************************************************************/

produce_subject_statistics: procedure expose newtopic_content true noflag blank newline subjectlist. subjecttotal. subject_q. subject_nq. version

  parse arg table_limit ; if table_limit = "" then table_limit = 0 ; total_entries = 0

  do i = 1 to subjectlist.0
    subjectname = subjectlist.i

    select
      when subject_q.subjectname  = 0 then subject_qs.subjectname = 100
      when subject_nq.subjectname = 0 then subject_qs.subjectname = 0

      otherwise subject_qs.subjectname = (subject_nq.subjectname/(subject_nq.subjectname+subject_q.subjectname))*100
    end

  end

  /* Write the message header. */

  if subjectlist.0 ~= 0 then do
    report_header = '       | Echomail Content (T:'subjectlist.0') |'

    report =  blank||newline'       +'||right(blank,(length(report_header)-9),"-")||'+'
    report = report||newline||report_header
    report = report||newline'  +----+'||right(blank,(length(report_header)-9),"-")||'+'||right(blank,(73-length(report_header)),"-")||'+'
    report = report||newline'  |  No.   Subject                                      Total       New  |'
    report = report||newline'  +----------------------------------------------------------------------+'

    /* Sort entries according to naughtiness in quoting. */

    'progressopen title "'version'" prompt "Sorting Subject Statistics..."'
     preq = result ; current_entry = 1

    do loop = 1 to subjectlist.0
      subject = 0 ; highest = 0

      do u = 1 to subjectlist.0
        subjectname = subjectlist.u

        if subjecttotal.subjectname ~< highest & subject_used.subjectname ~= "Null" then do
          subject = subjectname ; highest = subjecttotal.subjectname
        end
      end

      /* Work out the dominant flag, if there is one. */

      select
        when noflag = true then flag = " "
        when subject_qs.subject < newtopic_content then flag = "!"

        otherwise flag = " "
      end

      if flag = "!" then flag_! = true

      if (loop > table_limit) & table_limit ~= 0 & flag = " " then
        do 
          /* Do nothing */
        end
      else
        if (loop > table_limit) & table_limit ~= 0 then
          do 
            report = report||newline"  | "||right(loop,3," ")||". "||flag||" "||left(subject||" ",44,".")||right(" "||subjecttotal.subject||" ",7,".")||right(left((" "||subject_qs.subject||".0"),((index((" "||subject_qs.subject||".0"),".",1))+1)),8,".")||"%"||"  |"
            extra_entries = true
          end
        else
            report = report||newline"  | "||right(loop,3," ")||". "||flag||" "||left(subject||" ",44,".")||right(" "||subjecttotal.subject||" ",7,".")||right(left((" "||subject_qs.subject||".0"),((index((" "||subject_qs.subject||".0"),".",1))+1)),8,".")||"%"||"  |"

      subject_used.subject = "Null"

      /* Check to see if we've reached the desired table size. */

      if (loop = table_limit) & table_limit ~= 0 then do
        report = report||newline'  +----------------------------------------------------------------------+'
        table_limit_seperator = true
      end

      /* Update the progress requester. */

      'progressupdate 'preq' current 'loop' total 'subjectlist.0

      if rc = 5 then do
        'progressclose 'preq ; exit
      end

    end

    if table_limit_seperator ~= true | extra_entries = true then do
      report = report||newline'  +----------------------------------------------------------------------+'
    end

    if flag_! = true then do
      report = report||newline'  |  NB. ! = Thread is currently wasting bandwidth.                      |'
      report = report||newline'  +----------------------------------------------------------------------+'
    end

    'progressclose ' preq

  end

return report

/***************************************************************************/
/* Produce_Node_Statistics(). List originating nodes, with names.          */
/***************************************************************************/

produce_node_statistics: procedure expose true blank newline noflag address. userlist. writer. version

  parse arg table_limit ; if table_limit = "" then table_limit = 0 ; nodelist.0 = 0

  /* Grab the node information from the address. list. */

  if userlist.0 ~= 0 then do

    'progressopen title "'version'" prompt "Sorting Node Traffic..."'
     preq = result ; current_entry = 1

    do u = 1 to userlist.0
      username = userlist.u

      if address.username ~= "?" then do
        nodeaddress_result = left((address.username),((index((address.username),".",1))-1))

        if nodetotal.nodeaddress_result = 'NODETOTAL.'nodeaddress_result then
          do
            nodelist.0 = nodelist.0 + 1 ; nodetotal = nodelist.0
            nodelist.nodetotal = nodeaddress_result ; nodetotal.nodeaddress_result = writer.username
          end
        else 
          nodetotal.nodeaddress_result = nodetotal.nodeaddress_result + writer.username
      end

      /* Update the progress requester. */

       total = ((userlist.0)*2)
      'progressupdate 'preq' current 'u' total 'total

       if rc = 5 then do
         'progressclose 'preq ; exit
       end

    end  
  end

  if nodelist.0 ~= 0 then do
    report_header ='       | Echomail Origins (T:'nodelist.0') |'

    report =  blank||newline'       +'||right(blank,(length(report_header)-9),"-")||'+'
    report = report||newline||report_header
    report = report||newline'  +----+'||right(blank,(length(report_header)-9),"-")||'+'||right(blank,(73-length(report_header)),"-")||'+'
    report = report||newline'  |  No.   Name                          Address                  Total  |'
    report = report||newline'  +----------------------------------------------------------------------+'

    /* Sort according to amount of mail originating. */

    do loop = 1 to nodelist.0
      node = 0 ; highest = 0

      do u = 1 to nodelist.0
        nodename = nodelist.u

        if nodetotal.nodename ~< highest & node_used.nodename ~= "Null" then do
          node = nodename ; highest = nodetotal.nodename
        end
      end

      /* Check to see if the node is listed in the nodelist. */

      system_name = display_traplist_info(node)

      /* Work out the flags. */

      select
        when noflag = true then flag = " "
        when system_name = "?" then flag = "#"

        otherwise flag = " "
      end

      if flag = "#" then flag_# = true

      if (loop > table_limit) & table_limit ~= 0 & flag = " " then
        do 
          /* Do nothing */
        end
      else
        if (loop > table_limit) & table_limit ~= 0 then
          do 
            report = report||newline"  | "||right(loop,3," ")||". "||flag||" "||left(system_name||" ",29,".")||left(" "||node||" ",19,".")||right(" "||nodetotal.node,12,".")||"  |"
            extra_entries = true
          end
        else
          report = report||newline"  | "||right(loop,3," ")||". "||flag||" "||left(system_name||" ",29,".")||left(" "||node||" ",19,".")||right(" "||nodetotal.node,12,".")||"  |"

      node_used.node = "Null"

      /* Check to see if we've reached the desired table size. */

      if (loop = table_limit) & table_limit ~= 0 then do
        report = report||newline'  +----------------------------------------------------------------------+'
        table_limit_seperator = true
      end

      /* Update the progress requester. */

       current = int(((nodelist.0)/2)+(loop/2))
      'progressupdate 'preq' current 'current' total 'nodelist.0

       if rc = 5 then do
         'progressclose 'preq ; exit
       end

    end

    if table_limit_seperator ~= true | extra_entries = true then do
      report = report||newline'  +----------------------------------------------------------------------+'
    end

    if flag_# = true then do
      report = report||newline'  |  NB. # = Address is not present in the current nodelist.             |'
      report = report||newline'  +----------------------------------------------------------------------+'
    end

    'progressclose 'preq

  end

return report

/* ----------------------------------------------------------------------- */

/***************************************************************************/
/* Display_TrapList_Info(). Does what it says...                           */
/***************************************************************************/

display_traplist_info:

  parse arg nodenumber

  /* Get the info. */

  result = findnode(nodenumber,'System')
  parse var result success " " name

  /* Check if the address was present. */

  if success ~= 1 then do
    name = "?"
  end

return name

/***************************************************************************/
/* Int(). Returns an an interger from a specified number.                  */
/***************************************************************************/

int: procedure

  parse arg number

  if index(number,".") ~= 0 then
    result = left(number,(index(number,".")-1))
  else
    result = number

return result