import re

res = re.search(r'www\.(.*)\.(.{3})', 'www.baidu.com', re.I)  # 忽略大小写

if res:
    print(res.groups())  # 从分组1算起

    print("分组0：")
    print(res.group())
    print(res.group(0))

    print("分组1：")
    print(res.group(1))
    print(res.start(1))
    print(res.end(1))
    print(res.span(1))

    print("分组2：")
    print(res.group(2))
    print(res.start(2))
    print(res.end(2))
    print(res.span(2))

# 结果
'''
('baidu', 'com')
分组0：
www.baidu.com
www.baidu.com
分组1：
baidu
4
9
(4, 9)
分组2：
com
10
13
(10, 13)
'''

import re

res = re.search(r'baidu', 'www.baidu.com', re.I)
if res:
    print("search匹配成功")
    print(res.group())
else:
    print("search匹配失败")

rem = re.match(r'baidu', 'www.baidu.com', re.I)
if rem:
    print("match匹配成功")
    print(rem.group())
else:
    print("match匹配失败")

# 结果
'''
search匹配成功
baidu
match匹配失败
'''

import re

pattern = re.compile(r'\w+', re.I)
res = pattern.search('www.baidu.com')
if res:
    print("compile search匹配成功")
    print(res.group(0))
    print(res.start(0))
    print(res.end(0))
    print(res.span(0))
else:
    print("compile search匹配失败")

rem = pattern.match('www.baidu.com', 4, 8)
if rem:
    print("compile match匹配成功")
    print(rem.group(0))
    print(rem.start(0))
    print(rem.end(0))
    print(rem.span(0))
else:
    print("compile match匹配失败")

# 结果
'''
compile search匹配成功
www
0
3
(0, 3)
compile match匹配成功
baidu
4
9
(4, 9)
'''

import re

res = re.findall(r'\w+', 'Hello World!', re.I)
print(res)  # 返回的是一个列表

# 结果
'''
['Hello', 'World']
'''

# 通过compile编译正则表达式
pattern = re.compile(r'\w+', re.I)
rec = re.findall(pattern, 'Hello World!')
print(rec)

# 结果
'''
['Hello', 'World']
'''

import re

res0 = re.sub(r'\D', '', 'abc123#￥@', 0, re.I)  # 把非数字部分删除
res1 = re.sub(r'\D', '', 'abc123#￥@', 1, re.I)
res2 = re.sub(r'\D', '', 'abc123#￥@', 3, re.I)
print(res0)
print(res1)
print(res2)

# 结果
'''
123
bc123#￥@
123#￥@
'''

# 通过compile编译正则表达式
pattern = re.compile(r'\D', re.I)
res = re.sub(pattern, '', 'abc123#￥@', 0)  # 这时，不能传入flags匹配模式，而要在compile里传入
print(res)
# 结果
'''
123
'''

# repl是一个函数时，将匹配到的数字乘以2
def double(num):
    print(num.group('value'))
    value = int(num.group('value'))
    return str(value * 2)

result = re.sub(r'(?P<value>\d+)', double, 'a1B23C456d7890')
print(result)

# 结果
'''
1
23
456
7890
a2B46C912d15780
'''

import re

res1 = re.split('\W+', 'www.baidu.com')
res2 = re.split('(\W+)', 'www.baidu.com')
res3 = re.split('\W+', 'www.baidu.com', 1)  # 分割一次
res4 = re.split('p*', 'www.baidu.com')  # 匹配不到的正则表达式
print(res1)
print(res2)
print(res3)
print(res4)

# 结果
'''
['www', 'baidu', 'com']
['www', '.', 'baidu', '.', 'com']
['www', 'baidu.com']
['', 'h', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '']
'''

import re

res = re.escape('www.python.org')
print(res)

result = re.findall(re.escape('.py'), "python www.python.org proxy.py")
print(result)

# 结果
'''
www\.python\.org
['.py', '.py']
'''

import re

email_addr = 'Please reply this email to <abc23@sample.com.cn>'
# tool.chinaz.com给出的是：\w[-\w.+]*@([A-Za-z0-9][-A-Za-z0-9]+\.)+[A-Za-z]{2,14}
pattern = re.compile(r'([\w\.-]+)@([\w\.-]+)(\.[\w\.]+)')
res = re.search(pattern, email_addr)
if res:
    print(res.group)
else:
    print("匹配失败")

# 结果
'''
abc23@sample.com.cn
'''

import tesserocr
from PIL import Image


# 识别验证码图片转换为文本
image = Image.open("Verify.jpg")
result = tesserocr.image_to_text(image)
print(result)


# 直接将验证码图片转换为字符串（效果不如上者）
print(tesserocr.file_to_text("Verify.jpg"))

import tesserocr
from PIL import Image

image = Image.open('code.jpg')
image = image.convert('L')  # 图片转灰度

# image = image.convert('1')  # 二值化处理

threshold = 127  # 指定二值化的阈值
table = []
for i in range(256):
    if i < threshold:
        table.append(0)
    else:
        table.append(1)

image = image.point(table, '1')
result = tesserocr.image_to_text(text)
print(result)

# 创建项目
scrapy startproject test

# 创建一个Spider
cd test
scrapy genspider baidu www.baidu.com  # 网址是所要爬取网页的域名

import scrapy

class BaiduSpider(scrapy.Spider):
    name = "baidu"
    allowed_domains = ["www.baidu.com"]
    start_urls = ["http://www.baidu.com"]

    def parse(self, response):  # 这里提取方式用CSS选择器或者XPath选择器
        res = response.css('.quote')
        for s in res:
            item = BaiduItem()
            text = s.css('.text::text').extract_first()
            author = s.css('.author::text').extract_first()
            tags = s.css('.tags tag::text').extract()
            yield item

# 创建 Item，定义类型为scrapy.Field
import scrapy

class BaiduItem(scrapy.Item):
    text = scrapy.Field()
    author = scrapy.Field()
    tags = scrapy.Field()

# 找到后续的Request（即翻页）
next = response.css('.pager .next a::attr(href)').extract_first()
url = response.urljoin(next)
yield scrapy.Request(url=url, callback=self.parse)

# 运行项目
scrapy crawl baidu

# 保存文件
scrapy crawl baidu -o baidu.jl  # 保存为一行json
scrapy crawl baidu -o baidu.csv  # 保存为csv
scrapy crawl baidu -o baidu.xml  # 保存为xml

# 使用Item Pipeline实现保存到数据库，修改pipeline.py
from scrapy.exceptions import DropItem
import pymongo

class MongoPipeline(object):
    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri = crawler.setting.get("MONGO.URI"),
            mongo_db = crawler.setting.get("MONGO.DB")
        )

    def open_spider(self, spider):  # 初始化操作
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def process_item(self, item, spider):
        name = item.__class__.__name__
        self.db[name].insert(dict(item))
        return item

    def close_spider(self, spider):  # 关闭数据库链接
        self.client.close()

# 在setting.py中加入
ITEM_PIPELINES = {
    'test.pipilines.TextPipeline': 300,  # 保存为txt
    'test.pipilines.MongoPipeline': 400,  # 保存到mongodb
}
MONGO_URI = 'localhost'
MONGO_DB = test

myMpider/
    scrapy.cfg
    mySpider/
        __init__.py
        items.py
        pipelines.py
        settings.py
        middlewares.py
        spiders/
            __init__.py
            ...

# -*- coding: utf-8 -*-
# Define here the models for your scraped items
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy
from scrapy.item import Item, Field

class PropertiesItem(scrapy.Item):
    # Primary fields
    title = Field()
    price = Field()
    description = Field()
    address  = Field()
    image_urls = Field()

    # Calculated fields
    images = Field()
    location = Field()

    # Housekeeping fields
    url = Field()
    project = Field()
    spider = Field()
    server = Field()
    date = Field()

# 几种项目管道的方法：
def process_item(self, item, spider):
    return item   #返回的数据类型是dict类型的

def open_spider(self, spider):
        self.file = open('items.jl', 'w')

def close_spider(self, spider):
        self.file.close()

# 把item保存为json
import json

class JsonWriterPipeline(object):

    def open_spider(self, spider):
        self.file = open('items.jl', 'w')

    def close_spider(self, spider):
        self.file.close()

    def process_item(self, item, spider):
        line = json.dumps(dict(item)) + "\n"
        self.file.write(line)
        return item

# 保存到MongoDB数据库
import pymongo

class MongoPipeline(object):

    collection_name = 'scrapy_items'

    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGO_DATABASE', 'items')
        )

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        self.db[self.collection_name].insert_one(dict(item))
        return item

# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals

class PropertiesSpiderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, dict or Item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Response, dict
        # or Item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

class PropertiesDownloaderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        return None

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

# -*- coding: utf-8 -*-
import scrapy


class MyspiderSpider(scrapy.Spider):
    name = 'myspider'
    allowed_domains = ['www.baidu.com']
    start_urls = ['http://www.baidu.com/']

    def parse(self, response):
        pass