#
# Intended Platform: ASR 9k
# Author Vlad Ungureanu (vungurea@cisco.com)
# HOC:
#  25.02.2015 | vld
#   - initial script creation
#  27.02.2015 | vld
#   - added debug prints with env variable
#  03.03.2015 | vld
#   - changed cmd to filter output
#   - added $bd_name to the syslog message
#  05.03.2015 | vld
#   - changed saving of context to file
#   - removed script stormcontrol-drop-contextclear.tcl
#   - added exit command at the end
#   - added fix for function cli_read_pattern
#   - added redirect to file and read file to avoid limitation in IObuffers
#  10.03.2015 | vld
#   - changed file location to internal shared memory /tmp directory
#   - changed context saving file location to shared mem /tmp directory
#   - eliminated the need to specify destination files
#  28.05.2015 | vld
#   - removed plog for interface reset detection
#
# Author Tim Dorssers (tim.dorssers@vosko.nl)
# HOC:
#  02.11.2016 | tim
#   - added penalty point counting of drop threshold exceeding ACs
#   - added env option to shut down penalty threshold exceeding ACs or main IFs
#   - added env option to exclude main interfaces from being shut down
#   - added log message when drop threshold is no longer exceeded
#  14.11.2016 | tim
#   - fixed bug in function cli_read_pattern
#   - eliminated saving of array keys with zero penalty points
#   - added pseudo wire drop counter monitoring
#   - added warning if script runtime is close to maxrun
#   - set maxrun to polling interval to prevent multiple script instances
#   - changed DropValues array building to only include ACs with state up
#   - fixed bug in function subtract that caused miscalculations
#   - removed plog for drop counter reset detection
#  13.12.2016 | tim
#   - fixed plog unknown unicast PW traffic exceed
#  24.01.2017 | tim
#   - really fixed bug in function subtract that caused miscalculations
#  16.06.2017 | tim
#   - added script name to debug output and verbosity levels
#   - added commit error handling
#   - switched from watchdog to cron timer
#
# #############################################################################
#
# This script will monitor the drop count over a predefined time interval. If 
# the drops will exceed a certain threshold value then a syslog message will
# be generated. The command used for measuring drop counts is "show l2vpn
# bridge-domain detail"
#
#  To configure the parameters for the script please set the following in env:
#  EEM_DROP_CHECK_INTERVAL   - 60 120 180 240 300 360 600 720 900 1200 1800 s
#  [EEM_DROP_THRESHOLD]      - number of dropped packets (for BUM traffic)
#  [EEM_DROP_THRESHOLD_B]    - threshold value for broadcast traffic
#  [EEM_DROP_THRESHOLD_M]    - threshold value for multicast traffic
#  [EEM_DROP_THRESHOLD_UU]   - threshold value for unknown unicast traffic
#  [EEM_DROP_ACTION_SHUT]    - can be "ac", "main" or "none"
#  [EEM_DROP_THRESHOLD_SHUT] - penalty threshold for shutdown action
#  [EEM_DROP_EXCLUDE]        - main interfaces to be excluded from shutdown
#  [EEM_DROP_PRINTDEBUGINFO] - just set it to print debugs or set to "verbose"
#
# If you don't specify a value for the thresholds the default will be 100p
# The default shut down action is "none" (only monitoring). The default penalty
# threshold value is 3 penalty points
#
# #############################################################################
#

#Timer to repeat the script execution
::cisco::eem::event_register_timer cron name DropCheckTimer cron_entry [format "*/%s * * * *" [expr int($EEM_DROP_CHECK_INTERVAL / 60)]] maxrun $EEM_DROP_CHECK_INTERVAL

namespace import ::cisco::eem::*
namespace import ::cisco::lib::*

#namespace import ::cisco::fm::*

# errorInf gets set by namespace if any of the auto_path directories do not
# contain a valid tclIndex file.
set errorInfo ""

# Check the existence of input variables
if {![info exists EEM_DROP_CHECK_INTERVAL]} {
  set result "Mandatory EEM_DROP_CHECK_INTERVAL variable has not been set"
  error $result $errorInfo
} else {
  if {[lsearch {60 120 180 240 300 360 600 720 900 1200 1800} $EEM_DROP_CHECK_INTERVAL] < 0} {
    set result "Mandatory EEM_DROP_CHECK_INTERVAL variable is invalid"
    error $result $errorInfo
  }
}

# If not explicitly exported set default to 100 packets
if {![info exists EEM_DROP_THRESHOLD]} {
  set EEM_DROP_THRESHOLD 100
}
# If not explicitly exported set default to EEM_DROP_THRESHOLD
if {![info exists EEM_DROP_THRESHOLD_B]} {
  set EEM_DROP_THRESHOLD_B $EEM_DROP_THRESHOLD
}
if {![info exists EEM_DROP_THRESHOLD_M]} {
  set EEM_DROP_THRESHOLD_M $EEM_DROP_THRESHOLD
}
if {![info exists EEM_DROP_THRESHOLD_UU]} {
  set EEM_DROP_THRESHOLD_UU $EEM_DROP_THRESHOLD
}

# If not explicitly exported set default to none and validate string
if {![info exists EEM_DROP_ACTION_SHUT]} {
  set EEM_DROP_ACTION_SHUT "none"
} else {
  set EEM_DROP_ACTION_SHUT [string tolower $EEM_DROP_ACTION_SHUT]
  if {[lsearch {main ac} $EEM_DROP_ACTION_SHUT] < 0} {
    set result "Optional EEM_DROP_ACTION_SHUT variable is invalid"
    error $result $errorInfo
  }
}

# If not explicitly exported set default to 3 penalties
if {![info exists EEM_DROP_THRESHOLD_SHUT]} {
  set EEM_DROP_THRESHOLD_SHUT 3
}

# If not explicitly exported set default to empty
if {![info exists EEM_DROP_EXCLUDE]} {
  set EEM_DROP_EXCLUDE ""
}

set DEBUG_ENABLED 0
if {[info exists EEM_DROP_PRINTDEBUGINFO]} {
  if {[string match "verbose" [string tolower $EEM_DROP_PRINTDEBUGINFO]]} {
    set DEBUG_ENABLED 2
  } else {
    set DEBUG_ENABLED 1
  }
}

if {![info exists EEM_DROP_CMD_TEMPFILE]} {
  set EEM_DROP_CMD_TEMPFILE "stormcontrol-drop-monitor-output.txt"
}
# put the CMD_TEMPFILE always in the /tmp directory
set EEM_DROP_CMD_TEMPFILE "/tmp/$EEM_DROP_CMD_TEMPFILE"
set EEM_DROP_CMD_TEMPFILE_PREV "$EEM_DROP_CMD_TEMPFILE.prev"
set EEM_DROP_PENALTY_FILE "/tmp/stormcontrol-drop-monitor-penalty.txt"

# keep the time of starting the script
set startTime [clock seconds]

# will be set if we enter configuration mode of the router
set enterconfig 0

# script name
set scriptName [file tail $argv0]

# official cisco fix for function cli_read_pattern
# tim: moved trim_last_line out of while loop to fix duplicate lines
proc ::cisco::eem::cli_read_pattern {fd ptn} {
    set result ""
    set is_end 0
    set str2 ""
    set trim_last_line ""
    set i 1
    set last_line ""
    set line2 ""
    # lets not spin forever. try for maximum 50000 times
    while {$is_end == 0 && $i<=50000} {
        set str [read $fd]
        if {$str == ""} {
            after 50
            incr i;
            continue
        } else {
            #sometimes read returns router prompt partially.
            #we hope in at least 2 read() calls we get full router prompt
            set last_line $line2
            append last_line $str
            set line2 $str
        }
        # double quotes (don't change to curly braces!)
        set is_end [regexp "(.*)?($ptn)" $last_line]
        append result $str
        after 60
    }
    #remove last line containing routername
    set trim_last_line [regexp "(.*)\n" $result str2]
    if {$trim_last_line} {
        set result $str2
    }
    return $result
}

# print debug information
proc pdbg {dbg_level line_to_print} {
  global DEBUG_ENABLED
  global scriptName
  if {$DEBUG_ENABLED >= $dbg_level} {
    puts "$scriptName: $line_to_print"
  }
}

# read an array variable from a file
proc get_context_from_file {tmpfile context_data} {
  upvar 1 $context_data data
  if {[file exists $tmpfile]} {
    if {![catch {open $tmpfile "r"} fid] } {
      set buffer [read $fid]
      close $fid
      # parse the buffer and import array to memory
      foreach line [split $buffer "\n"] {
        #ignore empty lines
        if {0 == [string length [string trim $line]]} {
          continue
        }
        set key [lindex $line 0]
        set value [lrange $line 1 end]
        set data($key) "$value"
      }
      return 0
    } else {
      # error when opening file
      return -2
    }
  } else {
    # file does not exist
    return -1
  }
}

# save an array variable to a file
proc set_context_to_file {tmpfile context_data} {
  upvar 1 $context_data data
  if {![catch {open $tmpfile "w"} fid]} {
    foreach key [array names data] {
      puts $fid "$key $data($key)"
    }
    close $fid
    return 0
  }
  return -1
}

# define two helper functions
# this procedure will compare two large numbers represented as a list of digits
proc comp {n1l n2l len} {
  set i 0
  while {$i<$len} {
    set n1i [lindex $n1l $i]
    set n2i [lindex $n2l $i]
    if {$n1i == $n2i} {
      incr i 1
      continue
    }
    if {$n1i > $n2i} {
      return 1
    } else {
      return -1
    }
  }
  return 0
}

# this procedure will subtract two big positive numbers and will 
# return the result as a string
# tim: fixed broken string zero padding
proc subtract {n1s n2s} {
  #determine len and pad front with 0
  set n1l [string length $n1s]
  set n2l [string length $n2s]
  set nlen [expr ($n2l > $n1l) ? $n2l : $n1l]
  set n1 [split [format "%0$nlen\s" $n1s] ""]
  set n2 [split [format "%0$nlen\s" $n2s] ""]
  set n ""

  set cr [comp $n1 $n2 $nlen]
  if {$cr == 0} {
    return "0"
  }

  #iterate over list and make subtraction
  set t 0
  while {$nlen>0} {
    incr nlen -1
    set d1 [lindex $n1 $nlen]
    set d2 [lindex $n2 $nlen]
    if {$cr > 0} {
      set rx [expr $d1-$d2-$t]
    } else {
      set rx [expr $d2-$d1-$t]
    }
    if {$rx < 0} {
      set rx [expr $rx+10]
      set t 1
    } else {
      set t 0
    }
    set n $rx$n
  }
  # cut all trailing zeroes from the number
  set n [string trimleft $n "0"]
  # add sign if <0
  if {$cr == -1} {
    set n "-$n"
  }
  return $n
}

proc parse_cmd_output_file_and_build_array  {inputFile arrayOut} {
  upvar 1 $arrayOut DropValues
  array set DropValues []
  set pat [ list "Bridge group: (.*?), bridge-domain: (.*?)," \
   "    AC: (.*?), state is up" \
   "        packets: broadcast (\[0-9\]+), multicast (\[0-9\]+), unknown unicast (\[0-9\]+)" \
   "    PW: neighbor (.*?), PW ID (.*?),"
  ]
  set cur_bd_name ""
  set cur_bd_group ""
  set ac_intf ""
  set pw_neighbor ""
  set pw_id ""
  if {[file exists $inputFile]} {
    if {![catch {open $inputFile "r"} fid]} {
      set cmdOutput [read $fid]
      close $fid
      set cmdOut_array [split $cmdOutput "\n"]
      foreach line $cmdOut_array {
        # find bridge domain and bridge name
        if [regexp -linestop -- [lindex $pat 0] $line -> bd_group bd_name] {
          set ac_intf ""
          set cur_bd_name $bd_name
          set cur_bd_group $bd_group
          set pw_neighbor ""
        }
        # find the AC lines
        if [regexp -linestop -- [lindex $pat 1] $line -> ac_intf] {
          set pw_neighbor ""
        }
        # find dropped packets on AC
        if {([string compare $ac_intf ""] != 0) && \
            ([regexp -linestop -- [lindex $pat 2] $line -> drop_b drop_m drop_uu])} {
          pdbg 2 "Group: $cur_bd_group  Bridge-domain: $cur_bd_name  Interface: $ac_intf  b: $drop_b  m: $drop_m  uu: $drop_uu"
          # build a simple data structure with the values
          set DropValues($cur_bd_name:$ac_intf) "$drop_b $drop_m $drop_uu"
          set ac_intf ""
        }
        # find the PW lines
        if [regexp -linestop -- [lindex $pat 3] $line -> pw_neighbor pw_id] {
          set ac_intf ""
        }
        # find dropped packets on PW
        if {([string compare $pw_neighbor ""] != 0) && \
            ([regexp -linestop -- [lindex $pat 2] $line -> drop_b drop_m drop_uu])} {
          pdbg 2 "Group: $cur_bd_group  Bridge-domain: $cur_bd_name  PW neighbor: $pw_neighbor  PW ID: $pw_id  b: $drop_b  m: $drop_m  uu: $drop_uu"
          regsub -all {:} $pw_id {;} pw_id
          # build a simple data structure with the values
          set DropValues($cur_bd_name:$pw_neighbor:$pw_id) "$drop_b $drop_m $drop_uu"
          set pw_neighbor ""
        }
      }
      return 0
    } else {
      # error when opening file
      return -2
    }
  } else {
    # file does not exist
    return -1
  }
}

# sends interface shutdown command to cli and enters config mode if needed
proc shutdown_port {cli ac_intf} {
  global enterconfig
  upvar $cli cli1
  if {$enterconfig==0} {
    if [catch {cli_write $cli1(fd) "config"} result] {
       error $result $errorInfo
    }
    set enterconfig 1
  }
  cli_write $cli1(fd) "interface $ac_intf shutdown"
}

# increment an array, create if it doesn't exist
proc incrArrayElement {var key {incr 1}} {
  upvar $var a
  if {[info exists a($key)]} {
    incr a($key) $incr
  } else {
    set a($key) $incr
  }
}

# Read penalties from previous run
array set penalty {}
get_context_from_file $EEM_DROP_PENALTY_FILE penalty

# Open a cli session and execute the command
if [catch {cli_open} result] {
  error $result $errorInfo
} else {
  array set cli $result
}
# redirect the output of the command to a file
cli_exec $cli(fd) {terminal width 0}
cli_exec $cli(fd) {terminal length 0}
# make sure the output file does not exist
if {[file exists $EEM_DROP_CMD_TEMPFILE]} {
  file delete $EEM_DROP_CMD_TEMPFILE
}
# run the command and redirect output to file
set result [cli_exec $cli(fd) "run l2vpn_show -d 0x9 | grep -E \"bridge-domain: |AC: |packets: br|PW: nei\"> $EEM_DROP_CMD_TEMPFILE"]

# get the current drop values from the file
set ret [parse_cmd_output_file_and_build_array $EEM_DROP_CMD_TEMPFILE DropValues]
if {$ret != 0} {
  pdbg 1 "Error when trying to parse file $EEM_DROP_CMD_TEMPFILE"
  # Close CLI session
  if [catch {cli_close $cli(fd) $cli(tty_id)} result] {
    error $result $errorInfo
  }
  exit -1
}

pdbg 2 "tcl version is: $tcl_version"

# Compare the current values with the saved values
set ret [parse_cmd_output_file_and_build_array $EEM_DROP_CMD_TEMPFILE_PREV savedDropValues]
if {$ret == 0} {
  # compare values
  foreach key [array names savedDropValues] {
    if {[info exists DropValues($key)]} {
       # get the delta values to be compared
       set db [subtract [lindex $DropValues($key) 0] [lindex $savedDropValues($key) 0]]
       set dm [subtract [lindex $DropValues($key) 1] [lindex $savedDropValues($key) 1]]
       set duu [subtract [lindex $DropValues($key) 2] [lindex $savedDropValues($key) 2]]
       # split the key
       set key_array [split $key ":"]
       set bd_name [lindex $key_array 0]
       # are we looking at an AC or a PW
       if {[llength $key_array] > 2} {
         # key contains PW
         set pw_neighbor [lindex $key_array 1]
         set pw_id [lindex $key_array 2]
         regsub -all {;} $pw_id {:} pw_id_sub
         pdbg 2 "***$pw_neighbor $pw_id_sub $db $dm $duu***"
         # For Broadcast traffic
         if {($db > $EEM_DROP_THRESHOLD_B)} {
           # Broadcast traffic exceeded x pps in the last y seconds
           incrArrayElement penalty b:$pw_neighbor:$pw_id
           action_syslog priority notice msg "Broadcast traffic drop exceeded threshold $EEM_DROP_THRESHOLD_B\p/$EEM_DROP_CHECK_INTERVAL\s on $bd_name PW neighbor: $pw_neighbor, PW ID: $pw_id_sub Exact value: $db"
         } else {
           # Broadcast traffic below threshold
           if {[info exists penalty(b:$pw_neighbor:$pw_id)]} {
             unset penalty(b:$pw_neighbor:$pw_id)
             action_syslog priority notice msg "Broadcast traffic drop is below threshold $EEM_DROP_THRESHOLD_B\p/$EEM_DROP_CHECK_INTERVAL\s on $bd_name PW neighbor: $pw_neighbor, PW ID: $pw_id_sub"
           }
         }
         # For Multicast traffic
         if {($dm > $EEM_DROP_THRESHOLD_M)} {
           # Multicast traffic exceeded x pps in the last y seconds
           incrArrayElement penalty m:$pw_neighbor:$pw_id
           action_syslog priority notice msg "Multicast traffic drop exceeded threshold $EEM_DROP_THRESHOLD_M\p/$EEM_DROP_CHECK_INTERVAL\s on $bd_name PW neighbor: $pw_neighbor, PW ID: $pw_id_sub Exact value: $dm"
         } else {
           # Multicast traffic below threshold
           if {[info exists penalty(m:$pw_neighbor:$pw_id)]} {
             unset penalty(m:$pw_neighbor:$pw_id)
             action_syslog priority notice msg "Multicast traffic drop is below threshold $EEM_DROP_THRESHOLD_B\p/$EEM_DROP_CHECK_INTERVAL\s on $bd_name PW neighbor: $pw_neighbor, PW ID: $pw_id_sub"
           }
         }
         # For Unknown Unicast traffic
         if {($duu > $EEM_DROP_THRESHOLD_UU)} {
           # Unknown unicast traffic exceeded x pps in the last y seconds
           incrArrayElement penalty uu:$pw_neighbor:$pw_id
           action_syslog priority notice msg "Unknown unicast traffic drop exceeded threshold $EEM_DROP_THRESHOLD_UU\p/$EEM_DROP_CHECK_INTERVAL\s on $bd_name PW neighbor: $pw_neighbor, PW ID: $pw_id_sub Exact value: $duu"
         } else {
           # Unknown unicast traffic below threshold
           if {[info exists penalty(uu:$pw_neighbor:$pw_id)]} {
             unset penalty(uu:$pw_neighbor:$pw_id)
             action_syslog priority notice msg "Unknown unicast traffic drop is below threshold $EEM_DROP_THRESHOLD_B\p/$EEM_DROP_CHECK_INTERVAL\s on $bd_name PW neighbor: $pw_neighbor, PW ID: $pw_id_sub"
           }
         }
       } else {
         # key contains AC
         set ac_intf [lindex $key_array 1]
         pdbg 2 "***$ac_intf $db $dm $duu***"
         # For Broadcast traffic
         if {($db > $EEM_DROP_THRESHOLD_B)} {
           # Broadcast traffic exceeded x pps in the last y seconds
           incrArrayElement penalty b:$ac_intf
           if {![string match "none" $EEM_DROP_ACTION_SHUT]} {
             action_syslog priority notice msg "Broadcast traffic drop exceeded threshold $EEM_DROP_THRESHOLD_B\p/$EEM_DROP_CHECK_INTERVAL\s on $bd_name $ac_intf Exact value: $db Penalty: $penalty(b:$ac_intf)"
           } else {
             action_syslog priority notice msg "Broadcast traffic drop exceeded threshold $EEM_DROP_THRESHOLD_B\p/$EEM_DROP_CHECK_INTERVAL\s on $bd_name $ac_intf Exact value: $db"
           }
         } else {
           # Broadcast traffic below threshold
           if {[info exists penalty(b:$ac_intf)]} {
             unset penalty(b:$ac_intf)
             action_syslog priority notice msg "Broadcast traffic drop is below threshold $EEM_DROP_THRESHOLD_B\p/$EEM_DROP_CHECK_INTERVAL\s on $bd_name $ac_intf"
           }
         }
         # For Multicast traffic
         if {($dm > $EEM_DROP_THRESHOLD_M)} {
           # Multicast traffic exceeded x pps in the last y seconds
           incrArrayElement penalty m:$ac_intf
           if {![string match "none" $EEM_DROP_ACTION_SHUT]} {
             action_syslog priority notice msg "Multicast traffic drop exceeded threshold $EEM_DROP_THRESHOLD_M\p/$EEM_DROP_CHECK_INTERVAL\s on $bd_name $ac_intf Exact value: $dm Penalty: $penalty(m:$ac_intf)"
           } else {
             action_syslog priority notice msg "Multicast traffic drop exceeded threshold $EEM_DROP_THRESHOLD_M\p/$EEM_DROP_CHECK_INTERVAL\s on $bd_name $ac_intf Exact value: $dm"
           }
         } else {
           # Multicast traffic below threshold
           if {[info exists penalty(m:$ac_intf)]} {
             unset penalty(m:$ac_intf)
             action_syslog priority notice msg "Multicast traffic drop is below threshold $EEM_DROP_THRESHOLD_M\p/$EEM_DROP_CHECK_INTERVAL\s on $bd_name $ac_intf"
           }
         }
         # For Unknown Unicast traffic
         if {($duu > $EEM_DROP_THRESHOLD_UU)} {
           # Unknown unicast traffic exceeded x pps in the last y seconds
           incrArrayElement penalty uu:$ac_intf
           if {![string match "none" $EEM_DROP_ACTION_SHUT]} {
             action_syslog priority notice msg "Unknown unicast traffic drop exceeded threshold $EEM_DROP_THRESHOLD_UU\p/$EEM_DROP_CHECK_INTERVAL\s on $bd_name $ac_intf Exact value: $duu Penalty: $penalty(uu:$ac_intf)"
           } else {
             action_syslog priority notice msg "Unknown unicast traffic drop exceeded threshold $EEM_DROP_THRESHOLD_UU\p/$EEM_DROP_CHECK_INTERVAL\s on $bd_name $ac_intf Exact value: $duu"
           }
         } else {
           # Unknown unicast traffic below threshold
           if {[info exists penalty(uu:$ac_intf)]} {
             unset penalty(uu:$ac_intf)
             action_syslog priority notice msg "Unknown unicast traffic drop is below threshold $EEM_DROP_THRESHOLD_UU\p/$EEM_DROP_CHECK_INTERVAL\s on $bd_name $ac_intf"
           }
         }
       }
    } else {
      # the value for the bridge group existed and now does not exist
      # it means the interface went down or the bridge went down
      set key_array [split $key ":"]
      if {[llength $key_array] > 2} {
        # key contains PW
        set pw_neighbor [lindex $key_array 1]
        set pw_id [lindex $key_array 2]
        regsub -all {;} $pw_id {:} pw_id_sub
        pdbg 1 "Warning: Detected reset of PW neighbor: $pw_neighbor, PW ID: $pw_id_sub"
        # Cleanup penalties
        catch {unset penalty(b:$pw_neighbor:$pw_id)}
        catch {unset penalty(m:$pw_neighbor:$pw_id)}
        catch {unset penalty(uu:$pw_neighbor:$pw_id)}
      } else {
        # key contains AC
        set ac_intf [lindex $key_array 1]
        pdbg 1 "Warning: Detected reset of interface $ac_intf"
        # Cleanup penalties
        catch {unset penalty(b:$ac_intf)}
        catch {unset penalty(m:$ac_intf)}
        catch {unset penalty(uu:$ac_intf)}
      }
    }
  }
} elseif {$ret == -1} {
  # the file does not exist
  # this is the first time we execute the script so we just initialize
} elseif {$ret == -2} {
  # error occurred when opening file for reading even if file exists
  # rare case but good to see in debugs
  pdbg 1 "ERROR opening file $EEM_DROP_CMD_TEMPFILE_PREV for reading"
}

# shutdown interfaces if threshold exceeded
if {[string match "ac" $EEM_DROP_ACTION_SHUT]} {
  # Total AC penalties (broadcast+multicast+unknown unicast)
  array set total_penalty {}
  foreach key [array names penalty] {
    set key_array [split $key ":"]
    if {[llength $key_array] == 2} {
      set ac_intf [lindex $key_array 1]
      incrArrayElement total_penalty $ac_intf $penalty($key)
    }
  }
  foreach key [array names total_penalty] {
    set main_if [lindex [split $key "."] 0]
    if {$total_penalty($key) > $EEM_DROP_THRESHOLD_SHUT && [lsearch -regexp $EEM_DROP_EXCLUDE "^$main_if\$"] == -1} {
      # Main interface is not in the exclude list. Shutdown AC because of penalty threshold exceeded
      shutdown_port cli $key
      action_syslog priority notice msg "Penalty threshold ($EEM_DROP_THRESHOLD_SHUT) exceeded for $key, shutting down AC"
      # Cleanup penalties
      catch {unset penalty(b:$key)}
      catch {unset penalty(m:$key)}
      catch {unset penalty(uu:$key)}
    }
  }
} elseif {[string match "main" $EEM_DROP_ACTION_SHUT]} {
  # Total AC penalties (ACs per main interface and BUM types)
  array set main_penalty {}
  foreach key [array names penalty] {
    set key_array [split $key ":"]
    if {[llength $key_array] == 2} {
      set ac_intf [lindex $key_array 1]
      set main_if [lindex [split $ac_intf "."] 0]
      incrArrayElement main_penalty $main_if $penalty($key)
    }
  }
  foreach key [array names main_penalty] {
    if {$main_penalty($key) > $EEM_DROP_THRESHOLD_SHUT && [lsearch -regexp $EEM_DROP_EXCLUDE "^$key\$"] == -1} {
      # Main interface is not in the exclude list. Shutdown main interface because of penalty threshold exceeded
      shutdown_port cli $key
      action_syslog priority notice msg "Penalty threshold ($EEM_DROP_THRESHOLD_SHUT) exceeded for $key, total penalty: $main_penalty($key), shutting down main interface"
      # Cleanup penalties
      set main_if "*$key.*"
      foreach key [array names penalty] {
        if {[string match $main_if $key]} {
          unset penalty($key)
        }
      }
    }
  }
}

# if we have entered configuration mode, then commit
if {$enterconfig==1} {
  # Commit configuration, try 10 times
  for {set i 0} {$i < 10} {incr i} {
    if [catch {cli_write $cli1(fd) "commit"} result] {
      error $result $errorInfo
    }
    if [catch {cli_read_drain $cli1(fd) } result] {
      error $result $errorInfo
    } else {
      # Other configuration sessions?
      if {[string match "*One or more commits*" $result]} {
	    if [catch {cli_write $cli1(fd) "yes"} result] {
          error $result $errorInfo
        } else {
          break
	   }
      }
      # Configuration locked?
      if {![string match "*% Failed to commit*" $result]} break
    }
  }
}
# Close CLI session
if [catch {cli_close $cli(fd) $cli(tty_id)} result] {
    error $result $errorInfo
}

# make the current values the previous values by renaming file to .prev
# move API function is not implemented for the /tmp shared memory filesystem
# so we will use delete and copy
if {[file exists $EEM_DROP_CMD_TEMPFILE_PREV]} {
  file delete $EEM_DROP_CMD_TEMPFILE_PREV
}
catch {file copy $EEM_DROP_CMD_TEMPFILE $EEM_DROP_CMD_TEMPFILE_PREV}
if {[file exists $EEM_DROP_CMD_TEMPFILE]} {
  file delete $EEM_DROP_CMD_TEMPFILE
}

# save penalties for next run
set_context_to_file $EEM_DROP_PENALTY_FILE penalty

set endTime [clock seconds]
set totalRunTime [expr $endTime - $startTime]
pdbg 1 "Total script runtime was: $totalRunTime seconds"

if {$totalRunTime > [expr $EEM_DROP_CHECK_INTERVAL * 0.9]} {
  action_syslog priority notice msg "Warning: Runtime ($totalRunTime seconds) close to EEM policy interval ($EEM_DROP_CHECK_INTERVAL seconds)" 
}

exit