Python3网络爬虫开发实战阅读笔记

标签：age 爬虫 笔记 result print import message data Python3

基本库的使用

网络请求库

urllib（HTTP/1.1）

Python自带请求库，繁琐

基础使用：略

requests（HTTP/1.1）

Python常用第三方请求库，便捷

基础使用：略

httpx（HTTP/2.0）

Python第三方库，支持HTTP/2.0，支持异步请求，支持Python的async请求模式

pip install 'httpx[http2]'

基础使用：与requests相似，默认使用的是HTTP/1.1，需要开启HTTP/2.0

import httpx

client = httpx.Clien(http2=True)
response = client.get('url')
print(response.text)
client.close()

import httpx

headers={
    'User-Agent':'my-app/0.0.1'
}
with httpx.Client(headers=headers) as client:
    response = client.get('https://www.httpbin.org/get')
    print(response.json())

异步请求

import httpx
import asyncio

async def fetch(url):
    async with httpx.AsyncClient(http2=True) as client:
        response = await client.get(url)
        print(response.text)
        
if __name__ == '__main__':
    asyncio.get_event_loop().run_until_complete(fetch('url'))

aiohttp（异步）

import aiohttp
import asyncio

CONCURRENCY = 5

semaphore = asyncio.Semaphore(CONCURRENCY)
session = aiohttp.ClientSession()  #创建对象

async def get():
    params = {'name':'zhangsan','age':25}
    async with semaphore:
        async with session.get('https://www.httpbin.org/get',params=params) as response:
            print(await response.text())

if __name__ == '__main__':
    '''
    tasks = [asyncio.ensure_future(get()) for _ in range(10)]
    loop = asyncio.get_event_loop()
    loop.run_until_complete(asyncio.wait(tasks))
    '''
    asyncio.get_event_loop().run_until_complete(get())

    # asyncio.run(get())

网页数据解析

正则表达式

贪婪模式：.*

非贪婪模式：.*?

Tips：在表达式中，用（）来标注要取的位置，未标注表示忽略

修饰符

修饰符	描述
re.I	使匹配对大小不敏感
re.L	实现本地化识别（locale-aware）匹配
re.M	多行匹配，影响^和$
re.S	使匹配内容包括换行符在内的所有字符
re.U	根据Unicode字符集解析字符。影响\w、\W、\b、\B
re.X	灵活的书写格式

lxml

from lxml import etree

html = etree.HTML('text') #从文本中解析
html = etree.parse('index.html',etree.HTMLParser()) #从本地文件中解析
result = html.xpath('//')
print(result)

Beautiful Soup

from bs4 import BeautifulSoup

soup = BeautifulSoup('html','lxml')
print(soup)

方法选择器

from bs4 import BeautifulSoup

soup = BeautifulSoup('html','lxml')
result = soup.find_all(name='div',attrs={'id':'xxxx'})  #匹配对应标签和属性
result = soup.find_all(class_='xxx')  #匹配对应的属性名
result = soup.find_all(text=re.compile('link'))  #使用正则匹配包含link的标签
print(result.string) #输出标签内文本

CSS选择器

from bs4 import BeautifulSoup

soup = BeautifulSoup('html','lxml')
result = soup.select('css')
print(result['id'])  #输出属性id
print(result.attrs['id'])  #输出属性id
print(result.get_text())  #输出标签内文本
print(result.string)  #输出标签内文本

pyquery

找出来的不是列表，需要通过items()

from pyquery import PyQuery as pq

doc = pq('html')   #通过html创建对象
doc = pq(url='url')   #通过url创建对象，自动请求
doc = pq(filename='index.html')   #通过本地文件创建对象，自动请求
print(doc('li'))  #css选择
for item in doc('#id .list li').items():
    print(item.text())  #取出标签内文本

item=doc('ul')
lis = item.find('li')  #找出ul下所有的li标签
lis = item.children('li') #找出ul下所有子节点
item.parent()  #找出父节点标签
print(item.siblings('.xxx')) #找出兄弟节点

item.attr('href') #获取属性
item.attr.href  #获取属性

print(item.text())  #获取文本
item.html()

#节点操作
item.addClass('xxx')  #添加class
item.removeClass('xxx') #移除class
item.attr('name','link')  #修改/添加属性
item.text('修改文本')
item.html('html')
item.find('p').remove() #移除节点

#伪类选择器
li = doc('li:first-child')       #获取第一个节点
li = doc('li:last-child')        #获取最后一个节点
li = doc('li:nth-child(2)')      #获取第二个节点
li = doc('li:gt(2)')             #获取第三个li之后的li节点
li = doc('li:nth-child(2n)')     #获取偶数位置的li节点
li = doc('li:contains(second)')  #获取包含second文本的节点

parsel

同时支持xpath和css选择器，还支持正则，Scrapy的底层支持

from parsel import Selector

selector = Selector(text='html')
items = selector.css('li')  #css选择器
items2 = selector.xpath('//li[contains(@class,"item-0")]')  #xpath
for item in items:
    text = item.xpath('.//text()').get()
    print(text)
print(items2.getall())  #获取所有对应的文本，返回列表

result = selector.css('.item-0 *::text').getall()  #等于上面xpath语法

#提取属性
result = selector.css('.item-0::attr(href)').get()  #取对应class的属性
result = selector.xpath('//li[contains(@class,"item-0")]/a/@href')  #取包含指定class的href

#正则提取
result = selector.css('.item').re('link.*')  #匹配所有标签html

数据的存储

TXT

略

JSON

略

CSV

写入

import csv
#普通写入
with open('data.csv','w') as csvfile:
    writer = csv.writer(csvfile,[delimiter=' '])  #delimiter为分隔符，可不写
    writer.writerow(['id','name','age'])  #表头
    writer.writerow(['1000','zhuke','19'])  #写入一行
    writer.writerows([['1001','zhangsan','20'],['1002','lisi','21']])  #写入多行

import csv
#字典写入
with open('data.csv','a',encoding='utf-8') as csvfile:  #a为追加写入，encoding编码支持中文
    fieldnames = ['id','name','age']
    writer = csv.DictWriter(csvfile,fieldnames=fieldnames)  #fieldnames为表头
    writer.writerow({
        'id':'10003',
        'name':'王五',
        'age':22
    })

读取

import csv

with open('data.csv','r',encoding='utf-8') as csvfile:  #a为追加写入，encoding编码支持中文
    reader = csv.reader(csvfile)
    for row in reader:
        print(row)

MYSQL

关系型数据库

import pymysql

db = pymysql.connect(host='localhost',user='root',password='123456',port=3306)
cursor = db.cursor()  #获取操作游标
cursor.execute('SELECT VERSION()')
data = cursor.fetchone()  #获取第一条数据
print('Database version:',data)
cursor.execute('CREATE DABABASE spiders DEFAULT CHARACTER SET utf8mb4')  
db.close()

插入数据

import pymysql

db = pymysql.connect(host='localhost',user='root',password='123456',port=3306,db='spiders')  #指定数据库
cursor = db.cursor()  #获取操作游标
sql = 'CREATE TABLE IF NOT EXISTS students (id VARCHAR(255) NOT NULL,name= VARCHAR(255) NOT NULL,age INT NOT NULL,PRIMARY KEY(id))'
cursor.execute(sql)  #创建表
sql = 'INSERT INTO students(id,name,age) values(%ss,%ss,%ss)'
try:
    cursor.execute(sql,('1001','zhuke',18))  #插入数据
    cursor.commit()
except:
    db.rollback()
db.close()

通过字典插入

data = {
    'id':'1002',
    'name':'zhangsan',
    'age':20
}
table = 'students'
keys = ', '.join(data.keys())
values = ', '.join(['%s'] * len(data))
sql = 'INSERT INTO {table}({keys}) VALUES({values})'.format(table=table,keys=keys,values=values)
try:
    if cursor.execute(sql,tuple(data.values())):
        print('Successful')
        cursor.commit()
except:
    print('Failed', e)
    db.rollback()
db.close()

更新数据

import pymysql
data = {
    'id': '20120001',
    'name': 'Bob',
    'age': 21
}
table = 'students'
keys = ', '.join(data.keys())
values = ', '.join(['%s'] * len(data))
db = pymysql.connect(host='localhost', user='root',password=None, port=3306, db='spiders')
cursor = db.cursor()
sql = 'INSERT INTO {table}({keys}) VALUES ({values}) ON DUPLICATE KEY UPDATE '.format(
    table=table, keys=keys, values=values)
update = ','.join(["{key} = %s".format(key=key) for key in data])
sql += update
try:
    if cursor.execute(sql, tuple(data.values())*2):
        print('Successful')
        db.commit()
except Exception as e:
    print('Failed', e)
    db.rollback()
db.close()

删除数据

import pymysql

table = 'students'
condition = 'age > 20'

db = pymysql.connect(host='localhost', user='root', password=None, port=3306, db='spiders')
cursor = db.cursor()
sql = 'DELETE FROM  {table} WHERE {condition}'.format(table=table, condition=condition)
try:
    cursor.execute(sql)
    db.commit()
except:
    db.rollback()

db.close()

查询数据

import pymysql

sql = 'SELECT * FROM students WHERE age >= 20'

db = pymysql.connect(host='localhost', user='root', password=None, port=3306, db='spiders')
cursor = db.cursor()
try:
    cursor.execute(sql)
    print('Count:', cursor.rowcount)
    one = cursor.fetchone()
    print('One:', one)
    results = cursor.fetchall()
    print('Results:', results)
    print('Results Type:', type(results))
    for row in results:
        print(row)
except:
    print('Error')
    
sql = 'SELECT * FROM students WHERE age >= 20'
try:
    cursor.execute(sql)
    print('Count:', cursor.rowcount)
    row = cursor.fetchone()
    while row:
        print('Row:', row)
        row = cursor.fetchone()
except:
    print('Error')

MongoDB

非关系型数据库，键值存储数据库，性能非常高

import pymongo
#连接MongoDB
client = pymongo.MongoClient(host='localhost', port=27017)
# client = MongoClient('mongodb://localhost:27017/')

#指定数据库
db = client.test
# db = client['test']

#指定集合（表）
collection = db.students
# collection = db['students']

插入数据

直接返回id

student1 = {
    'id': '20170101',
    'name': 'Jordan',
    'age': 20,
    'gender': 'male'
}
student2 = {
    'id': '20170202',
    'name': 'Mike',
    'age': 21,
    'gender': 'male'
}
result = collection.insert(student1)  #插入一条数据
result = collection.insert([student1, student2]) #插入多条数据
print(result)

插入数据（最新）

返回对象

result = collection.insert_one(student1)  #插入一条数据
print(result.inserted_id)
result = collection.insert_many([student1, student2])  #插入多条数据
print(result)
print(result.inserted_ids)

查询数据

#查询一条数据
result = collection.find_one({'name': 'Mike'})
print(type(result))
print(result)
#查询多条数据
results = collection.find({'age': 20})
print(results)
for result in results:
    print(result)
#查询大于20的数据
results = collection.find({'age': {'$gt': 20}})
#正则匹配查询
results = collection.find({'name': {'$regex': '^M.*'}})  #查询以M开头的
#计数
count = collection.find().count()  #总条数
print(count)
count = collection.find({'age': 20}).count()  #条件计数
print(count)
#排序
results = collection.find().sort('name', pymongo.ASCENDING)  #升序
print([result['name'] for result in results])
results = collection.find().sort('name', pymongo.ASCENDING).skip(2)  #忽略前面两个元素
print([result['name'] for result in results])
results = collection.find().sort('name', pymongo.ASCENDING).skip(2).limit(2)  #限制取出的结果数量
print([result['name'] for result in results])

比较符号

符号	含义	实例
$lt	小于	{ 'age' : { '$lt' : 20 }}
$gt	大于	{ 'age' : { '$gt' : 20 }}
$lte	小于等于	{ 'age' : { '$lte' : 20 }}
$gte	大于等于	{ 'age' : { '$gte' : 20 }}
$ne	不等于	{ 'age' : { '$ne' : 20 }}
$in	在范围内	{ 'age' : { '$in' : [20,23] }}
$nin	不在范围内	{ 'age' : { '$nin' : [20,23] }}

更新数据

condition = {'name': 'Kevin'}
student = collection.find_one(condition)
student['age'] = 25
result = collection.update(condition, student)
print(result)

result = collection.update(condition, {'$set': student})

condition = {'name': 'Kevin'}
student = collection.find_one(condition)
student['age'] = 26
result = collection.update_one(condition, {'$set': student})
print(result)
print(result.matched_count, result.modified_count)

condition = {'age': {'$gt': 20}}
result = collection.update_one(condition, {'$inc': {'age': 1}})
print(result)
print(result.matched_count, result.modified_count)

condition = {'age': {'$gt': 20}}
result = collection.update_many(condition, {'$inc': {'age': 1}})
print(result)
print(result.matched_count, result.modified_count)

删除数据

result = collection.remove({'name': 'Kevin'})
print(result)
#官方推荐
result = collection.delete_one({'name': 'Kevin'})
print(result)
print(result.deleted_count)
result = collection.delete_many({'age': {'$lt': 25}})
print(result.deleted_count)

其他操作：find_one_and_delete、find_one_and_replace、find_one_and_update

Redis

菜鸟教程：点击查看

Elasticsearch

暂无

RabbitMQ

消息队列中间件

多进程

import multiprocessing

def main(page):
    pass
if __name__ == '__main__':
    pool = multiprocessing.Pool()  #创建进程池
    pages = range(1, TOTAL_PAGE+1)
    pool.map(main,pages)  #使用map方法main进行多进程传参
    pool.close()
    pool.join()

异步

import asyncio

CONCURRENCY = 5
semaphore = asyncio.Semaphore(CONCURRENCY)  #设置并发数

async def main():
    # index tasks
    global session
    session = aiohttp.ClientSession()
    scrape_index_tasks = [asyncio.ensure_future(scrape_index(page)) for page in range(1, PAGE_NUMBER + 1)]
    results = await asyncio.gather(*scrape_index_tasks) #添加到任务列表，返回结果是函数的返回列表
    # detail tasks
    print('results', results)
    ids = []
    for index_data in results:
        if not index_data: continue
        for item in index_data.get('results'):
            ids.append(item.get('id'))
    scrape_detail_tasks = [asyncio.ensure_future(scrape_detail(id)) for id in ids]
    await asyncio.wait(scrape_detail_tasks) #添加到任务列表，返回结果是对象
    await session.close()
if __name__ == '__main__':
    loop = asyncio.get_event_loop()
    loop.run_until_complete(main())

自动化框架

Selenium

Splash

非Python

Pyppeteer

支持异步

Playwright

安装

pip install playwright
palywright install

代码生成（录制）

Playwright官方网站

playwright codegen -o script.py -b firefox

PhantomJS

非Python

JavaScript逆向

JavaScript逆向技巧总结

1. 寻找入口
   • 查看请求（Network）
   • 搜索参数（Search）
   • 分析发起调用（XHR）
   • 断点（XHR、DOM、事件断点）、debugger
   • Hook（Base64编码、Cookie的赋值、JSON的序列化）、油猴、浏览器自带Override
   • 其他（使用Pyppeteer、Playwright里的API进行数据拦截，或者浏览器第三方插件）
2. 调试分析
   • 格式化代码（美化）
   • 断点调试
   • 反混淆（AST，控制流平坦化）
3. 模拟执行
   • Python改写或模拟执行（pyexecjs）
   • Javascript模拟执行+API（Node.js）
   • 浏览器模拟执行

安卓逆向

安卓APK调试（debuggable）

apk动态调试需要提前开启debuggable

通常有三种做法:

1. 修改程序的AndroidManifest.xml中的android:debuggable=”true”，然后签名重打包；

2. 修改系统属性，将defalut.prop中ro.debugable设置为1,利用mprop工具修改当前手机应用都可以调试；

3. 使用xposed模块：BDOpener——开启APK调试与备份选项的Xposed模块

   第二种方法详细操作

   1. 查看apk是否为可调式

   adb shell
   cat default.prop
   

   解释：ro.debuggable=0 为不可调式， 当值为1 为可调式

   2.查看cpu架构使用

   cat /proc/cpuinfo
   

   3.修改ro.debuggable的值

   adb push /xx/xx/mprop/data/local/tmp/mprop
   adb shell
   su(获取root权限)
   cd /data/local/tmp
   chmod 777 mprop
   ./mprop ro.debuggable 1
   

JEB动态调试

操作教程

网易MuMu模拟器

打开cmd（如果使用MuMu自带adb，则cd C:\Program Files (x86)\Nemu\vmonitor\bin\）

adb kill-server（MuMu自带：adb_server.exe kill-server）

连接模拟器端口：adb connect 127.0.0.1:7555（MuMu自带：adb_server.exe connect 127.0.0.1:7555）

列出已连接的设备：adb devices（MuMu自带：adb_server.exe devices），正常会显示MuMu的设备已连接，则可以进行下一步的操作了

adb kill-serve
adb connect 127.0.0.1:7555
adb shell am start -D -n com.goldze.mvvmhabit/.ui.MainActivity
adb shell am start -D -n （包名）/（.主窗体）

夜神模拟器

adb connect 127.0.0.1:62001
D:\Nox\bin\nox_adb.exe devices
D:\Nox\bin\nox_adb.exe shell am start -D -n com.qianyu.zhuceji/.MainActivity  (包名/Activity)

Xposed

AppHook框架

安卓低版本使用：Xposed

安卓高版本使用：EdXposed

安卓无root使用：VirtualXposed

与 Xposed 相比，目前 VirtualXposed 有两个限制：

1. 不支持修改系统（可以修改普通 APP 中对系统 API 的调用），因此重力工具箱，应用控制器等无法使用。
2. 暂不支持资源 HOOK，因此资源钩子不会起任何作用；使用资源 HOOK 的模块，相应的功能不会生效。

开发Xposed模块

AndroidStudio配置AndroidManifest.xml，application里面添加以下配置用来标识xposed模块

<meta-data
           android:name="xposedmodule"
           android:value="true" />
<meta-data
           android:name="xposeddescription"
           android:value="Xposed Test" />
<meta-data
           android:name="xposedminversion"
           android:value="53" />

引入Xposed SDK，在app/build.gradle文件中的dependencies里添加以下配置

compileOnly 'de.robv.android.xposed:api:82'
compileOnly 'de.robv.android.xposed:api:82:sources'

MainActivity.java

package com.meteor.xposedtest;

import androidx.appcompat.app.AppCompatActivity;
import android.os.Bundle;
import android.view.View;
import android.widget.Button;
import android.widget.Toast;

public class MainActivity extends AppCompatActivity {
    private Button button;
    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);
        button = findViewById(R.id.button);
        button.setOnClickListener(new View.OnClickListener(){
            public void onClick(View v){
                Toast.makeText(MainActivity.this,showMessage(1,2),Toast.LENGTH_SHORT).show();
            }
        });
    }

    public String showMessage(int x, int y) {
        return "x + y = " + (x + y);
    }
}

在MainActivity.java同级目录创建HookMessage.java

package com.meteor.xposedtest;

import de.robv.android.xposed.IXposedHookLoadPackage;
import de.robv.android.xposed.XC_MethodHook;
import de.robv.android.xposed.XposedBridge;
import de.robv.android.xposed.XposedHelpers;
import de.robv.android.xposed.callbacks.XC_LoadPackage;

public class HookMessage implements IXposedHookLoadPackage{
    public void handleLoadPackage(XC_LoadPackage.LoadPackageParam loadPackageParam) throws Throwable {
		//判断包名，指定app
        if (loadPackageParam.packageName.equals("com.meteor.xposedtest")){
            XposedBridge.log("Hooked com.meteor.xposedtest Package");
            //加载包中的类
            Class clazz = loadPackageParam.classLoader.loadClass("com.meteor.xposedtest.MainActivity");
            //在类中寻找方法并且hook，参数有多少个写多少个，参数类型.class
            XposedHelpers.findAndHookMethod(clazz, "showMessage", int.class, int.class, new XC_MethodHook() {
                //在调用之前hook，一般用来修改参数
                protected void beforeHookedMethod(MethodHookParam param) throws Throwable {
                    XposedBridge.log("Called beforeHookedMethod");
                    param.args[0] = 2;  //修改第一个参数的值
                    XposedBridge.log("Changed args 0 to " + param.args[0]);
                }
                //在方法执行后hook，一般用来保存、转发、拦截
                protected void afterHookedMethod(MethodHookParam param) throws Throwable {
                    XposedBridge.log("Called afterHookedMethod");
                    //param.setResult("Hooked");  //设置方法的返回结果
                    //param.getResult();  //获取返回结果
                }
            });
        }
    }
}

Xposed入口文件 在main文件夹下新建一个Assets Folder，在assets文件夹下新建一个xposed_init文件，没有后缀名，把hook的类的路径写进去

com.meteor.xposedtest.HookMessage

Frida

安装frida

pip install frida-tools

安卓需要安装 frida-server，下载地址，安装教程

连接夜神模拟器

adb connect 127.0.0.1:62001

启用frida-server

adb shell
cd /data/local/tmp
chmod 777 frida-ser
./frida-ser

转发连接

adb forward tcp:27042 tcp:27042

查看手机进程

frida-ps -U

Java层Hook

js代码

Java.perform(()=>{
    let MainActivity = Java.use('com.germey.appbasic1.MainActivity')
    console.log('start hook')
    MainActivity.getMessage.implementation = (arg1,arg2)=>{
        send('start hook')
        return '6'
    }
})

py代码

import frida
import sys

CODE = open('hook_java.js',encoding='utf-8').read()
PROCESS_NAME = 'AppBasic1'

def on_message(message,data):
    print(message)

process = frida.get_remote_device().attach(PROCESS_NAME)  #这里要传入进程名，用frida-ps -U可查看
script = process.create_script(CODE)
script.on('message',on_message)
script.load()
sys.stdin.read()

调用Java函数并且获取返回值

Java.perform(()=>{
    let MainActivity = Java.use('d.b.a.n.K')
    console.log('start hook')
    MainActivity.a.overload('java.lang.String', 'java.lang.String', 'long', 'int').implementation = (url,param,time,nonce)=>{
        send(param)
        var day_date = String(Date.parse(new Date(2022,0,12)));
        var cons = MainActivity.$new();  //重新new一个
        var solution = cons.a(url,param,time,nonce);
        var send_it = solution.toString();
        var data = `${send_it}--${time}--${nonce}--${day_date}`
        send(data)
        return ''
    }
})

hook组件代码示例

Java.perform(function () {
    var tv_class = Java.use("android.widget.TextView");
    tv_class.setText.overload("java.lang.CharSequence").implementation = function (x) {
        var string_to_send = x.toString();
        var string_to_recv;
        send(string_to_send); // 将数据发送给kali主机的python代码
        recv(function (received_json_object) {
            string_to_recv = received_json_object.my_data
            console.log("string_to_recv: " + string_to_recv);
        }).wait(); //收到数据之后，再执行下去
        return this.setText(string_to_recv);
    }
});

import time
import frida

def my_message_handler(message, payload):
    print message
    print payload
    if message["type"] == "send":
        print message["payload"]
        data = message["payload"].split(":")[1].strip()
        print 'message:', message
        data = data.decode("base64") # 解码
        user, pw = data.split(":") # 提取用户名和密码
        data = ("admin" + ":" + pw).encode("base64") # 组成新的组合并编码
        print "encoded data:", data
        script.post({"my_data": data})  # 将JSON对象发送回去
        print "Modified data sent"

device = frida.get_usb_device()
pid = device.spawn(["com.roysue.demo04"])
device.resume(pid)
time.sleep(1)
session = device.attach(pid)
with open("s4.js") as f:
    script = session.create_script(f.read())
script.on("message", my_message_handler)  # 注册消息处理函数
script.load()
raw_input()

Native层Hook

js代码

Java.perform(function(){
    Interceptor.attach(Module.findExportByName('libnative.so','Java_com_germey_appbasic2_MainActivity_getMessage'),{
        onEnter:function(args){
            send('hook onEnter')
            send('args[1]=' + args[2])
            send('args[2]=' + args[3])
        },
        onLeave:function(val){
            send('hook onLeave')
            val.replace(Java.vm.getEnv().newStringUtf('5'))  //替换值
        }
    })
})

py代码

import frida
import sys

CODE = open('hook_native.js',encoding='utf-8').read()
PROCESS_NAME = 'AppBasic2'

def on_message(message,data):
    print(message)

process = frida.get_remote_device().attach(PROCESS_NAME)
script = process.create_script(CODE)
script.on('message',on_message)
script.load()
sys.stdin.read()

RPC远程调用

import frida

def on_message(message, data):
    if message['type'] == 'send':
        print(message['payload'])
    elif message['type'] == 'error':
        print(message['stack'])

session = frida.get_usb_device().attach('com.roysue.roysueapplication')

source = """
    rpc.exports = {
    add: function (a, b) {
        return a + b;
    },
    sub: function (a, b) {
        return new Promise(function (resolve) {
        setTimeout(function () {
            resolve(a - b);
        }, 100);
        });
    }
    };
"""

script = session.create_script(source)
script.on('message', on_message)
script.load()
print(script.exports.add(2, 3))
print(script.exports.sub(5, 3))
session.detach()

详细示例

function callSecretFun() { //定义导出函数
    Java.perform(function () {
        //to-do 做自己想做的事情 
        //比如这里是找到隐藏函数并且调用
        Java.choose("com.roysue.demo02.MainActivity", {
            onMatch: function (instance) {
                console.log("Found instance: " + instance);
                console.log("Result of secret func: " + instance.secret());
            },
            onComplete: function () { }
        });
    });
}
rpc.exports = {
    callsecretfunction: callSecretFun //把callSecretFun函数导出为callsecretfunction符号，导出名不可以有大写字母或者下划线
};

import time
import frida

def my_message_handler(message, payload):
    print message
    print payload

device = frida.get_usb_device()
pid = device.spawn(["com.roysue.demo02"])
device.resume(pid)
time.sleep(1)
session = device.attach(pid)
with open("s3.js") as f:
    script = session.create_script(f.read())
script.on("message", my_message_handler)
script.load()

command = ""
while 1 == 1:
    command = raw_input("Enter command:n1: Exitn2: Call secret functionnchoice:")
    if command == "1":
        break
    elif command == "2": #在这里调用
        script.exports.callsecretfunction()

枚举所有加载类

setTimeout(function (){
  Java.perform(function (){
    console.log("n[*] enumerating classes...");
    //Java对象的API enumerateLoadedClasses
    Java.enumerateLoadedClasses({
      //该回调函数中的_className参数就是类的名称，每次回调时都会返回一个类的名称
      onMatch: function(_className){
        //在这里将其输出
        console.log("[*] found instance of '"+_className+"'");

        //如果只需要打印出com.roysue包下所有类把这段注释即可，想打印其他的替换掉indexOf中参数即可定位到~
        //if(_className.toString().indexOf("com.roysue")!=-1)
        //{
        //    console.log("[*] found instance of '"+_className+"'");
        //}
      },
      onComplete: function(){
        //会在枚举类结束之后回调一次此函数
        console.log("[*] class enuemration complete");
      }
    });
  });
});

枚举类中所有方法并定位

function enumMethods(targetClass)
{
    var hook = Java.use(targetClass);
    var ownMethods = hook.class.getDeclaredMethods();
    hook.$dispose;
    return ownMethods;
}

function hook_overload_5() {
    if(Java.available) {
        Java.perform(function () {
           var a = enumMethods("com.roysue.roysueapplication.User$clz")
           a.forEach(function(s) {
                console.log(s);
           });
        });
    }
}

拦截类的所有方法

function traceClass(targetClass)
{
    //Java.use是新建一个对象哈，大家还记得么？
    var hook = Java.use(targetClass);
    //利用反射的方式，拿到当前类的所有方法
    var methods = hook.class.getDeclaredMethods();
    //建完对象之后记得将对象释放掉哈
    hook.$dispose;
    //将方法名保存到数组中
    var parsedMethods = [];
    methods.forEach(function(method) {
        //通过getName()方法获取函数名称
        parsedMethods.push(method.getName());
    });
    //去掉一些重复的值
    var targets = uniqBy(parsedMethods, JSON.stringify);
    //对数组中所有的方法进行hook
    targets.forEach(function(targetMethod) {
        traceMethod(targetClass + "." + targetMethod);
    });
}
function hook_overload_9() {
    if(Java.available) {
        Java.perform(function () {
            console.log("start hook");
            traceClass("com.roysue.roysueapplication.Ordinary_Class");
            console.log("hook end");
        });
    }
}
s1etImmediate(hook_overload_9);

APK脱壳

frida_dump

填入包名进行脱壳，脱壳后的dex会在手机的/data/data/com.goldze.mvvmhabit/files文件夹里

frida -U -f com.goldze.mvvmhabit -l dump_dex.js --no-pause

将手机里的文件拉取到电脑中

adb pull /data/data/com.goldze.mvvmhabit/files ./dexes

frida-dexdump

frida-dexdump -n com.goldze.mvvmhabit -f

标签：age,爬虫,笔记,result,print,import,message,data,Python3
From： https://www.cnblogs.com/pigke/p/17487553.html