We can significance-check shellscripts.

diff --git a/filterator b/filterator
index b3f52abc8404931f3c50065c0e1e0aa013c4d85d..7ac184eccc6ee10c28136c0bf2323d33fc60ceab 100755 (executable)
--- a/filterator
+++ b/filterator
@@ -21,19 +21,33 @@ C_junk = map(re.compile, (
     r'enum', r'\bint\b', r'\blong\b', r'\bshort\b', r'\bstatic\b',
     r'\bstruct\b', r'typedef', r'\bunion\b', r'\bvoid\b',
     r'# *define',r'# *endif',r'# *else',r'# *if\b',
-    r'# *ifdef\b',r'# *ifndef\b', 
+    r'# *ifdef\b',r'# *ifndef\b',
+    # Comment delimiters with no content
     r'/\*+', r'\*+/', r'\*+',
     # Common preprocessor macros, not significant by themselves.
     r'\bASSERT\b', r'\bFALSE\b', r'\bNULL\b', r'\bSTATIC\b', r'\bTRUE\b',
     # Macro include lines are noise, too.
-    r'\s*#include.*',
+    r'\s*#include.*',r'#\s*line.*',
     # Common error macros.
-    r'\bEINVAL\b', r'\bENOSYS\b', 
+    r'\bEFAULT\b', r'\bEINVAL\b', r'\bENOSYS\b',
+    # Punctuation
+    r'\{', r'\}', r'\(', r'\)', r'\<', r'\>', r'\[', r'\]',
+    r'\^', '&', r'\|', r'\*', r'\?', r'\.', r'\+', 
+    '\\\n', ';', ':', '%', ',', '-', '/', '=', '!', '\n','\t', '\v'
+    ))
+shell_junk = map(re.compile, (
+    # Idioms that don't convey any meaning in isolation
+    r"exit *\([01]\);",
+    # Bare shell keywords
+    "\bcase\b", "\bdone\b", "\belse\b", "\besac\b", "\bfi\b", "\bif\b",
+    "\bwhile\b", 
+    # Blank comment
+    "^#\n",
     ))
 
-def nontrivial(text, filetype):
+def nontrivial(text, deletia):
     "Identify a shred as trivial or nontrivial."
-    if filetype == "C":
+    if deletia:
         # Basic theory of this function is that if we throw out all C
         # syntax and and common constants, and there is still an
         # identifier, we're looking at something that might be
@@ -41,24 +55,21 @@ def nontrivial(text, filetype):
         text = ' ' + text
         while True:
             savecopy = text
-            for regexp in C_junk:
+            for regexp in deletia:
                 text = regexp.sub(' ', text)
-            for ch in ('{','}', '(',')', '<','>', '[',']', '\\\n',
-                       ';', ':', ',', '.', '%', '^', '&', '|', '*','?',
-                       '+', '-', '/', '=', '!', '\n','\t', '\v'):
-                text = text.replace(ch, ' ')
-            if debug:
-                print "Reduced:", `text`
             if savecopy == text:
                 break
             else:
+                text = text.strip()
+                if debug:
+                    print "Reduced:", `text`
                 continue
     return text.strip()
 
-def deduce_filetype(line):
-    "Deduce filetype from the data on its first line."
+def deduce_deletia(line):
+    "Deduce deletia from the data on its first line."
     if "sh" in line:
-        return "sh"
+        return shell_junk
 
 def report_common():
     "Generate a report on common code."
@@ -76,26 +87,26 @@ def report_common():
                 if end - start + 1 < minsize:
                     continue
                 try:
-                    # Try to deduce the filetype so we know how to filter
-                    filetype = None
+                    # Try to deduce the deletia
+                    deletia = None
                     if file.endswith(".c") or file.endswith(".h"):
-                        filetype = "C"
+                        deletia = C_junk
                     # Skip to the relevant chunk
                     rfp = open(file)
                     for i in range(start-1):
                         line = rfp.readline()
-                        if not filetype and i == 0:
-                            filetype = deduce_filetype(line)
+                        if not deletia and i == 0:
+                            deletia = deduce_deletia(line)
                     text = ""
                     for i in range(start, end+1):
                         nextline = rfp.readline()
-                        if not filetype and i == 0 and start == 1:
-                            filetype = deduce_filetype(line)
+                        if not deletia and i == 0 and start == 1:
+                            deletia = deduce_deletia(line)
                         if nextline[0] == '%':
                             nextline = '%' + nextline
                         text += nextline
                     rfp.close()
-                    if nofilter or nontrivial(text, filetype):
+                    if nofilter or nontrivial(text, deletia):
                         # We do this here in order to avoid losing
                         # the error message in noise if the user needs
                         # -d and fails to supply it.