init project

rsshub/__init__.py

@@ -0,0 +1,61 @@
+import os
+import click
+from flask import Flask, render_template
+from flask.cli import with_appcontext
+from rsshub.config import config
+from rsshub.extensions import *
+from rsshub.blueprints.main import bp as main_bp
+from rsshub.utils import XMLResponse
+
+
+def create_app(config_name=None):
+    if config_name is None:
+        config_name = os.getenv('FLASK_CONFIG', 'development')
+
+    app = Flask(__name__)
+    app.config.from_object(config[config_name])
+    app.response_class = XMLResponse
+
+    register_blueprints(app)
+    register_extensions(app)
+    register_errors(app)
+    register_cli(app)
+
+    return app
+
+
+def register_extensions(app):
+    bootstrap.init_app(app)
+    debugtoolbar.init_app(app)
+    moment.init_app(app)
+
+
+def register_blueprints(app):
+    app.register_blueprint(main_bp)
+
+
+def register_errors(app):
+    @app.errorhandler(400)
+    def bad_request(e):
+        return render_template('errors/400.html'), 400
+
+    @app.errorhandler(404)
+    def page_not_found(e):
+        return render_template('errors/404.html'), 404
+
+    @app.errorhandler(500)
+    def internal_server_error(e):
+        return render_template('errors/500.html'), 500
+
+
+def register_cli(app):
+    @app.cli.command()
+    @with_appcontext
+    def ptshell():
+        """Use ptpython as shell."""
+        try:
+            from ptpython.repl import embed
+            if not app.config['TESTING']:
+                embed(app.make_shell_context())
+        except ImportError:
+            click.echo('ptpython not installed! Use the default shell instead.')

rsshub/blueprints/main.py

@@ -0,0 +1,46 @@
+from flask import Blueprint, render_template, current_app, request
+
+
+bp = Blueprint('main', __name__)
+
+
+@bp.route('/')
+def index():
+    return render_template('main/index.html')
+
+
+@bp.route('/feeds')
+def feeds():
+    feed_rules = [rule for rule in current_app.url_map._rules if 'main' in rule.endpoint][:-2]
+    return render_template('main/feeds.html', rules=feed_rules)
+
+
+@bp.app_template_global()
+def filter_content(ctx):
+    include_title = request.args.get('include_title')
+    include_description = request.args.get('include_description')
+    exclude_title = request.args.get('exclude_title')
+    exclude_description = request.args.get('exclude_description')
+    limit = request.args.get('limit', type=int)
+    items = ctx['items'].copy()
+    items = [item for item in items if include_title in item['title']] if include_title else items
+    items = [item for item in items if include_description in item['description']] if include_description else items
+    items = [item for item in items if exclude_title not in item['title']] if exclude_title else items
+    items = [item for item in items if exclude_description not in item['description']] if exclude_description else items
+    items = items[:limit] if limit else items
+    ctx = ctx.copy()
+    ctx['items'] = items
+    return ctx
+
+
+#---------- feed路由从这里开始 -----------#
+@bp.route('/guokr/scentific')
+def guokr_scientific():
+    from rsshub.spiders.guokr.scientific import ctx
+    return render_template('main/atom.xml', **filter_content(ctx))
+
+
+@bp.route('/toutiao/today')
+def toutiao_today():
+    from rsshub.spiders.toutiao.today import ctx
+    return render_template('main/atom.xml', **filter_content(ctx))

rsshub/config.py

@@ -0,0 +1,32 @@
+import os
+import sys
+
+
+basedir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+
+
+class BaseConfig:
+    SITE_NAME = 'RSSHub'
+    GITHUB_USERNAME = 'alphardex'
+    EMAIL = '2582347430@qq.com'
+    SECRET_KEY = os.environ.get('SECRET_KEY') or 'f43hrt53et53'
+    DEBUG_TB_INTERCEPT_REDIRECTS = False
+
+
+class DevelopmentConfig(BaseConfig):
+    pass
+
+
+class TestingConfig(BaseConfig):
+    TESTING = True
+
+
+class ProductionConfig(BaseConfig):
+    pass
+
+
+config = {
+    'development': DevelopmentConfig,
+    'testing': TestingConfig,
+    'production': ProductionConfig
+}

rsshub/extensions.py

@@ -0,0 +1,8 @@
+from flask_bootstrap import Bootstrap
+from flask_debugtoolbar import DebugToolbarExtension
+from flask_moment import Moment
+
+
+bootstrap = Bootstrap()
+debugtoolbar = DebugToolbarExtension()
+moment = Moment()

rsshub/spiders/guokr/scientific.py

@@ -0,0 +1,23 @@
+import requests
+
+data = requests.get(
+    'https://www.guokr.com/apis/minisite/article.json?retrieve_type=by_subject'
+).json()['result']
+
+
+def parse(d):
+    item = {}
+    item['title'] = d['title']
+    item['description'] = f"{d['summary']}<br><img referrerpolicy='no-referrer' src={d['image_info']['url']}"
+    item['pubDate'] = d['date_published']
+    item['link'] = d['url']
+    return item
+
+
+ctx = {
+    'title': '果壳网 科学人',
+    'link': 'https://www.guokr.com/scientific',
+    'description': '果壳网 科学人',
+    'author': 'alphardex',
+    'items': list(map(parse, data))
+}

rsshub/spiders/toutiao/today.py

@@ -0,0 +1,22 @@
+from rsshub.utils import fetch
+
+base_url = 'https://toutiao.io'
+tree = fetch(base_url)
+items = []
+posts = tree.css('.posts .post')
+
+for post in posts:
+    item = {}
+    item['title'] = post.css('.title a::attr(title)').extract_first()
+    item['description'] = ''
+    item['pubDate'] = ''
+    item['link'] = base_url + post.css('.title a::attr(href)').extract_first()
+    items.append(item)
+
+ctx = {
+    'title': '开发者头条:今日头条',
+    'link': 'https://toutiao.io',
+    'description': '开发者头条:今日头条',
+    'author': 'alphardex',
+    'items': items
+}

rsshub/static/css/style.css

@@ -0,0 +1,48 @@
+/* global */
+
+nav {
+    margin-bottom: 30px;
+}
+
+.jumbotron {
+    margin-top: 20px;
+    padding-top: 38px;
+    padding-bottom: 38px;
+}
+
+.tip {
+    /* from github.com */
+    position: relative;
+    padding: 40px;
+    text-align: center;
+    background-color: #fafbfc;
+    border: 1px solid #e1e4e8;
+    border-radius: 3px;
+    box-shadow: inset 0 0 10px rgba(27, 31, 35, 0.05);
+}
+
+.hide {
+    display: none;
+}
+
+.inline {
+    display: inline;
+}
+
+.page-header {
+    padding-top: 20px;
+    padding-bottom: 20px;
+}
+
+.page-footer {
+    padding-top: 40px;
+}
+
+
+/* footer */
+
+footer {
+    margin: 30px 0;
+    padding: 20px 0;
+    border-top: 1px solid #e5e5e5;
+}

rsshub/templates/errors/400.html

@@ -0,0 +1,9 @@
+{% extends 'layout.html' %}
+
+{% block title %}400 错误{% endblock %}
+
+{% block content %}
+<div class="jumbotron">
+    <h1>400 Bad Request</h1>
+</div>
+{% endblock %}

rsshub/templates/errors/404.html

@@ -0,0 +1,9 @@
+{% extends 'layout.html' %}
+
+{% block title %}404 错误{% endblock %}
+
+{% block content %}
+<div class="jumbotron">
+    <h1>404 Not Found</h1>
+</div>
+{% endblock %}

rsshub/templates/errors/500.html

@@ -0,0 +1,9 @@
+{% extends 'layout.html' %}
+
+{% block title %}500 错误{% endblock %}
+
+{% block content %}
+<div class="jumbotron">
+    <h1>服务器出错</h1>
+</div>
+{% endblock %}

rsshub/templates/layout.html

@@ -0,0 +1,69 @@
+{% from 'bootstrap/nav.html' import render_nav_item %}
+<!DOCTYPE html>
+<html>
+
+<head>
+    {% block head %}
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <title>{% block title %}{% endblock title %}</title>
+    <link rel="shortcut icon" href="{{ url_for('static', filename='favicon.ico') }}" type="image/x-icon">
+    <link rel="icon" href="{{ url_for('static', filename='favicon.ico') }}" type="image/x-icon">
+    {% block styles %}
+    {{ bootstrap.load_css() }}
+    <link rel="stylesheet" href="{{ url_for('static', filename='css/style.css') }}">
+    <link href="https://cdn.bootcss.com/font-awesome/4.7.0/css/font-awesome.min.css" rel="stylesheet">
+    {% endblock styles %}
+    {% endblock head %}
+</head>
+
+<body>
+    {% block nav %}
+    <nav class="navbar navbar-expand-lg navbar-dark bg-dark" role="navigation">
+        <div class="container">
+            <a class="navbar-brand" href="{{url_for('main.index')}}">{{ config['SITE_NAME'] }}</a>
+            <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarNavDropdown"
+                aria-controls="navbarNavDropdown" aria-expanded="false" aria-label="Toggle navigation">
+                <span class="navbar-toggler-icon"></span>
+            </button>
+            <div class="collapse navbar-collapse" id="navbarNavDropdown">
+                <ul class="nav navbar-nav ml-auto">
+                </ul>
+            </div>
+        </div>
+    </nav>
+    {% endblock nav %}
+    <main class="container">
+        {% with messages = get_flashed_messages(with_categories=true) %}
+        {% if messages %}
+        {% for category, message in messages %}
+        <div class="alert alert-dismissable alert-{{ category }}" role="alert">
+            <button type="button" class="close" data-dismiss="alert">&times;</button>
+            {{ message }}
+        </div>
+        {% endfor %}
+        {% endif %}
+        {% endwith %}
+        {% block content %}{% endblock %}
+        {% block footer %}
+        <footer>
+            <p class="float-left">
+                <small>&copy;
+                    Made by <a href="https://github.com/{{config['GITHUB_USERNAME']}}" target="_blank">{{config['GITHUB_USERNAME']}}</a>
+                    with <span class="fa fa-heart"></span>
+                </small>
+            </p>
+            <p class="float-right">
+                <small><a href="mailto:{{ config['EMAIL'] }}">Contact</a></small>
+            </p>
+        </footer>
+        {% endblock footer %}
+    </main>
+    {% block scripts %}
+    {{ bootstrap.load_js() }}
+    {{ moment.include_moment() }}
+    {{ moment.locale('zh-cn') }}
+    {% endblock %}
+</body>
+
+</html>

rsshub/templates/main/atom.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+    <generator>{{config['SITE_NAME']}}</generator>
+    <webMaster>{{config['EMAIL']}}</webMaster>
+    <language>zh-cn</language>
+    <id>{{link}}</id>
+    <title><![CDATA[{{title|safe}}]]></title>
+    <link href="{{link}}"/>
+    <author>
+        <name><![CDATA[{{author|safe}}]]></name>
+    </author>
+    {% for item in items %}
+        <entry>
+            <id>{{item.link}}</id>
+            <title><![CDATA[{{item.title|safe}}]]></title>
+            <published>{{item.pubDate}}</published>
+            <updated>{{item.pubDate}}</updated>
+            <link href="{{item.link}}"/>
+            <content type="html" src="{{item.link}}"><![CDATA[{{item.description|safe}}]]></content>
+        </entry>
+    {% endfor %}
+</feed>

rsshub/templates/main/feeds.html

@@ -0,0 +1,11 @@
+{% extends "layout.html" %}
+
+{% block title %}All Feeds{% endblock title %}
+
+{% block content %}
+<div class="list-group">
+    {% for rule in rules %}
+        <a href="{{rule.rule}}" class="list-group-item list-group-item-action">{{rule.endpoint[5:]}}</a>
+    {% endfor %}
+</div>
+{% endblock content %}

rsshub/templates/main/index.html

@@ -0,0 +1,13 @@
+{% extends 'layout.html' %}
+
+{% block title %}Welcome to RSShub!{% endblock title %}
+
+{% block content %}
+<div class="jumbotron">
+    <h1 class="display-4">Welcome to RSSHub!</h1>
+    <p class="lead">If you see this page, the RSSHub is successfully installed and working.</p>
+    <p>
+        <a class="btn btn-primary btn-lg" role="button" href="{{url_for('main.feeds')}}">View All Feeds</a>
+    </p>
+</div>
+{% endblock %}

rsshub/utils.py

@@ -0,0 +1,25 @@
+from flask import Response
+import requests
+from parsel import Selector
+
+DEFAULT_HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'}
+
+
+class XMLResponse(Response):
+    def __init__(self, response, **kwargs):
+        if 'mimetype' not in kwargs and 'contenttype' not in kwargs:
+            if response.startswith('<?xml'):
+                kwargs['mimetype'] = 'application/xml'
+        return super().__init__(response, **kwargs)
+
+
+def fetch(url: str, headers: dict=DEFAULT_HEADERS, proxies: dict=None):
+    try:
+        res = requests.get(url, headers=headers, proxies=proxies)
+        res.raise_for_status()
+    except Exception as e:
+        print(f'[Err] {e}')
+    else:
+        html = res.text
+        tree = Selector(text=html)
+        return tree