Merge branch 'v1.0.5' into master

Author: Your Name

README.md

@@ -25,22 +25,25 @@
 - Python3的运行环境
 
 ### 目录说明
+```
 AppInfoScanner
     |-- libs  程序的核心代码
         |-- core
             |-- parses.py 用于解析文件中的静态信息
         |-- task
-            |-- android_task.py 用于处理Android相关的文件
-            |-- ios_task.py 用于处理iOS相关的文件
-            |-- web_task.py 用于处理Web相关的文件，比如网页右键源代码、H5相关的静态信息
-    |-- tools 程序需要依赖的工具
-        |-- apktool.jar 用于反编译apk文件
-        |-- baksmali.jar 用于反编译dex文件
-        |-- strings.exe 用于windows 32下获取iPA的字符串信息
-        |-- strings64.exe 用于windows 64的系统获取iPA的字符串信息
-    |-- app.py 主运行程序
-    |-- config.py 用于自定义相关规则
-    |-- readme.md  程序使用说明
+            |-- base_task.py 统一任务调度
+ 			|-- android_task.py 用于处理Android相关的文件            
+​			 |-- ios_task.py 用于处理iOS相关的文件
+​            |-- web_task.py 用于处理Web相关的文件，比如网页右键源代码、H5相关的静态信息
+​    |-- tools 程序需要依赖的工具
+​        |-- apktool.jar 用于反编译apk文件
+​        |-- baksmali.jar 用于反编译dex文件
+​        |-- strings.exe 用于windows 32下获取iPA的字符串信息
+​        |-- strings64.exe 用于windows 64的系统获取iPA的字符串信息
+​    |-- app.py 主运行程序
+​    |-- config.py 用于自定义相关规则
+​    |-- readme.md  程序使用说明
+```
 
 
 
@@ -91,7 +94,7 @@ python3 app.py android -i <Your apk or dex directory>  -a
 
 ```
 python3 app.py android -i <Your apk or dex directory> -t 10
-``` 
+```
 
 ### iOS 相关操作说明
 

app.py

@@ -7,70 +7,67 @@
 import click
 
 from libs.core import Bootstrapper
-from libs.task.android_task import AndroidTask
-from libs.task.ios_task import iOSTask
-from libs.task.web_task import WebTask
+from libs.task.base_task import BaseTask
 
 @click.group(help="Python script for automatically retrieving key information in app.")
 def cli():
     pass
 
 # 创建Android任务
 @cli.command(help="Get the key information of Android system.")
-@click.option("-i", "--input", required=True, type=str, help="Input APK file or DEX directory.")
+@click.option("-i", "--inputs", required=True, type=str, help="Input APK file or DEX directory.")
 @click.option("-r", "--rules", required=False, type=str, default="", help="Add regular search rule.")
 @click.option("-s", "--net-sniffer", is_flag=True, default=False, help="Whether to enable network sniffing.")
 @click.option("-n", '--no-resource', is_flag=True, default=False,help="Ignore resource files.")
 @click.option("-p", '--package',required=False,type=str,default="",help="Specifies the retrieval package name.")
-@click.option("-a", '--all',is_flag=True, default=False,help="Output all strings.")
+@click.option("-a", '--all-str',is_flag=True, default=False,help="Output all strings.")
 @click.option("-t", '--threads',required=False, type=int,default=10,help="Set the number of threads to 10 by default")
-def android(input: str, rules: str, net_sniffer: bool,no_resource:bool,package:str,all:bool,threads:int) -> None:
+def android(inputs: str, rules: str, net_sniffer: bool,no_resource:bool,package:str,all_str:bool,threads:int) -> None:
     try:
         # 初始化全局对象
         bootstrapper = Bootstrapper(__file__)
         bootstrapper.init()
 
-        task = AndroidTask(input, rules, net_sniffer,no_resource,package,all,threads)
-        # 让内层代码直接抛出异常，不做rollback。
-        task.start()
+        BaseTask("Android", inputs, rules, net_sniffer, no_resource, package, all_str, threads).start()
     except Exception as e:
         raise e
 
 
 @cli.command(help="Get the key information of iOS system.")
-@click.option("-i", "--input", required=True, type=str, help="Input IPA file or ELF file.")
+@click.option("-i", "--inputs", required=True, type=str, help="Input IPA file or ELF file.")
 @click.option("-r", "--rules", required=False, type=str, default="", help="Add regular search rule.")
 @click.option("-s", "--net-sniffer", is_flag=True, default=False, help="Whether to enable network sniffing.")
 @click.option("-n", '--no-resource', is_flag=True, default=False,help="Ignore resource files.")
-@click.option("-a", '--all',is_flag=True, default=False,help="Output all strings.")
+@click.option("-a", '--all-str',is_flag=True, default=False,help="Output all strings.")
 @click.option("-t", '--threads',required=False, type=int,default=10,help="Set the number of threads to 10 by default")
-def ios(input: str, rules: str, net_sniffer: bool,no_resource:bool,all:bool,threads:int) -> None:
+def ios(inputs: str, rules: str, net_sniffer: bool,no_resource:bool,all_str:bool,threads:int) -> None:
     try:
         # 初始化全局对象
         bootstrapper = Bootstrapper(__file__)
         bootstrapper.init()
 
-        task = iOSTask(input, rules, net_sniffer,no_resource,all,threads)
-        # 让内层代码直接抛出异常，不做rollback。
-        task.start()
+        BaseTask("iOS", inputs, rules, net_sniffer, no_resource, all_str, threads).start()
+        
     except Exception as e:
         raise e
 
 
 @cli.command(help="Get the key information of Web system.")
-@click.option("-i", "--input", required=True, type=str, help="Input WebSite dir.")
+@click.option("-i", "--inputs", required=True, type=str, help="Input WebSite dir.")
 @click.option("-r", "--rules", required=False, type=str, default="", help="Add regular search rule.")
-@click.option("-a", '--all',is_flag=True, default=False,help="Output all strings.")
+@click.option("-a", '--all-str',is_flag=True, default=False,help="Output all strings.")
 @click.option("-t", '--threads',required=False, type=int,default=10,help="Set the number of threads to 10 by default")
-def web(input: str, rules: str, all:bool,threads:int) -> None:
+def web(inputs: str, rules: str, all_str:bool,threads:int) -> None:
     try:
         # 初始化全局对象
         bootstrapper = Bootstrapper(__file__)
         bootstrapper.init()
 
+        # BaseTask("Web", inputs, rules,all_str, threads).start()
+
         task = WebTask(input, rules,all,threads)
-        # 让内层代码直接抛出异常，不做rollback。
         task.start()
+
     except Exception as e:
         raise e
 

config.py

@@ -3,19 +3,33 @@
 # Github: https://github.com/kelvinBen/AppInfoScanner
 
 
-# 此处用于搜索组件信息，如fastjson、gson等
+# 此处用于搜索组件信息
+# com.alibaba.fastjson -> fastjson
+# com.google.gson -> gson
+# com.fasterxml.jackson -> jackson
+# net.sf.json -> 
+# javax.xml.parsers.DocumentBuilder -> dom方式
+# javax.xml.parsers.SAXParser -> sax方式
+# org.jdom.input.SAXBuilder -> jdom
+# org.dom4j.io.SAXReader -> dom4j
 filter_components = [
     'com.alibaba.fastjson',
     'com.google.gson',
     'com.fasterxml.jackson',
-    'net.sf.json'
+    'net.sf.json',
+    'javax.xml.parsers.DocumentBuilder',
+    'javax.xml.parsers.SAXParser',
+    'org.jdom.input.SAXBuilder',
+    'org.dom4j.io.SAXReader'
 ]
 
 # 此处目前支持过滤
 # 1. https://以及http://开头的
 # 2. IPv4的ip地址
 # 3. URI地址
 filter_strs =[
+    r'^http://.*',
+    r'^https://.*',
     r'.*(http://.*)',
     r'.*(https://.*)', 
     r'.*((?:[0-9]{1,3}\.){3}[0-9]{1,3}).*',

libs/core/__init__.py

@@ -1,10 +1,11 @@
 # -*- coding: utf-8 -*-
 # Author: kelvinBen
 # Github: https://github.com/kelvinBen/AppInfoScanner
-
-import platform
 import os
+import time
 import shutil
+import platform
+
 
 # smali 所在路径
 smali_path = ""
@@ -30,49 +31,42 @@ def __init__(self, path):
         global os_type
         global output_path
         global script_root_dir
-        global result_path
+        global txt_result_path
+        global xls_result_path
         global strings_path
 
+        create_time = time.strftime("%Y%m%d%H%M%S", time.localtime())
+
         script_root_dir =  os.path.dirname(os.path.abspath(path))
         tools_dir = os.path.join(script_root_dir,"tools")
-
+        
         if platform.system() == "Windows":
             machine2bits = {'AMD64':64, 'x86_64': 64, 'i386': 32, 'x86': 32}
             machine2bits.get(platform.machine())
 
             if platform.machine() == 'i386' or platform.machine() == 'x86':
-                strings_path = os.path.join(script_root_dir,"strings.exe")
+                strings_path = os.path.join(tools_dir,"strings.exe")
             else:
-                strings_path = os.path.join(script_root_dir,"strings64.exe")
+                strings_path = os.path.join(tools_dir,"strings64.exe")
         else:
             strings_path ="strings"
-        #     os_type = "win"
-        #     smali_str = "smali.bat"
-        #     back_smali_str = "backsmali.bat"
-        #     apktool_path_str = "apktool.bat"
-        # elif platform.system() == "Linux":
-        #     os_type = "lin"
-        #     smali_str = "smali"
-        #     back_smali_str = "backsmali"
-        #     apktool_path_str = "apktool"
-        # else:
-        #     os_type = "mac"
-        #     smali_str = "smali"
-        
 
-        # smali_path = os.path.join(tools_dir,str(os_type) + os.sep + smali_str)
         backsmali_path = os.path.join(tools_dir,"baksmali.jar")
         apktool_path = os.path.join(tools_dir, "apktool.jar")
         output_path = os.path.join(script_root_dir,"out")
-        result_path = os.path.join(script_root_dir,"result.txt")
+        txt_result_path = os.path.join(script_root_dir,"result_"+str(create_time)+".txt")
+        xls_result_path = os.path.join(script_root_dir,"result_"+str(create_time)+".xls")
 
     def init(self):
         if os.path.exists(output_path):
             shutil.rmtree(output_path)
         os.makedirs(output_path)
 
-        if os.path.exists(result_path):
-            os.remove(result_path)
+        if os.path.exists(txt_result_path):
+            os.remove(txt_result_path)
+
+        if os.path.exists(xls_result_path):
+            os.remove(xls_result_path)
 
 
 

libs/core/parses.py

@@ -2,72 +2,70 @@
 # Author: kelvinBen
 # Github: https://github.com/kelvinBen/AppInfoScanner
 
-
-import threading
-import config
 import re
 import os
+import config
+import threading
 import libs.core as cores
 
 class ParsesThreads(threading.Thread):
 
-    def __init__(self,threadID,name,file_queue,all,result_dict):
+    def __init__(self,threadID,name,file_queue,all,result_dict,types):
         threading.Thread.__init__(self) 
         self.file_queue = file_queue
         self.name = name
         self.threadID = threadID
         self.result_list = []
         self.all = all
         self.result_dict=result_dict
+        self.types = types
 
-    def __regular_parse__(self,threadLock):
+    def __regular_parse__(self):
         while True:
-            try:
-                file_path = self.file_queue.get(timeout = 5)
-                scan_str = ("Scan file : %s" % file_path)
-                print(scan_str)
-
-                try:
-                    os.path.basename(file_path).split(".")[1]
-                except Exception as e:
-                    self.__get_string__(file_path,threadLock)
-                    continue
-                self.__file_parse__(file_path,threadLock)
-
-                result_set =  set(self.result_list)
-                if len(result_set) !=0:
-                    self.result_dict[file_path] = result_set
-
-                if self.file_queue.empty():
-                    break
-            except Exception as e:
+            if self.file_queue.empty():
                 break
+            
+            file_path = self.file_queue.get(timeout = 5)
+            scan_str = ("Scan file : %s" % file_path)
+            print(scan_str)
+
+            if self.types == "iOS":
+                self.__get_string_by_iOS__(file_path)
+            else:
+                self.__get_string_by_file__(file_path)
+            
+            result_set =  set(self.result_list)
+            if len(result_set) !=0:
+                self.result_dict[file_path] = result_set
+
+    def __get_string_by_iOS__(self,file_path):
+        output_path = cores.output_path
+        strings_path = cores.strings_path
+        temp =  os.path.join(output_path,"temp.txt")
+        cmd_str = ("%s %s > %s") % (strings_path,file_path,temp)
+        if os.system(cmd_str) == 0:
+            with open(temp,"r",encoding='utf-8',errors='ignore') as f:
+                lines = f.readlines()
+                for line in lines:
+                    self.__parse_string__(line)
 
-    def __file_parse__(self,file_path,threadLock):
-        with open(file_path,"r",encoding="utf8") as file :
-            file_content =  file.read()
+    def __get_string_by_file__(self,file_path):
+        with open(file_path,"r",encoding="utf8",errors='ignore') as f :
+            file_content =  f.read()
             # 获取到所有的字符串
             pattern = re.compile(r'\"(.*?)\"') 
             results = pattern.findall(file_content)
 
             # 遍历所有的字符串
             for result in set(results): 
-                self.__parse_string__(result,threadLock)
-
-    def __get_string__(self,dir_file_path,threadLock):
-        temp =  os.path.join(cores.output_path,"temp.txt")
-        cmd_str = ("%s %s > %s") % (cores.strings_path,dir_file_path,temp)
-        if os.system(cmd_str) == 0:
-            with open(temp,"r") as f:
-                lines = f.readlines()
-                for line in lines:
-                    self.__parse_string__(line,threadLock)
-
-    def __parse_string__(self,result,threadLock):
+                self.__parse_string__(result)    
+        
+    def __parse_string__(self,result):
         # 通过正则筛选需要过滤的字符串
         for filter_str in config.filter_strs:
             filter_str_pat = re.compile(filter_str) 
             filter_resl = filter_str_pat.findall(result)
+            # print(result,filter_resl)
             # 过滤掉未搜索到的内容
             if len(filter_resl)!=0:
                 # 提取第一个结果
@@ -76,9 +74,9 @@ def __parse_string__(self,result,threadLock):
                 if self.__filter__(resl_str) == 0:
                     continue
 
-                threadLock.acquire()
-                self.result_list.append(filter_resl[0])
-                threadLock.release()
+                self.threadLock.acquire()
+                self.result_list.append(resl_str)
+                self.threadLock.release()
             continue
 
     def __filter__(self,resl_str):
@@ -105,5 +103,5 @@ def __filter__(self,resl_str):
         return return_flag  
 
     def run(self):
-        threadLock = threading.Lock()
-        self.__regular_parse__(threadLock)
+        self.threadLock = threading.Lock()
+        self.__regular_parse__()