Added RSS feeder (#92)

app.json

@@ -78,6 +78,11 @@
         "SPAM_CHECK_EXCEPTION_GROUPS": {
             "description": "Groups where spam scans shouldn't happen, leave it empty if you don't know what that is",
             "required": false
+        },
+        "RSS_DELAY": {
+            "description": "Delay in which RSS will send updates in chat",
+            "value": 300,
+            "required": true
         }
     }
 }

requirements.txt

@@ -21,3 +21,4 @@ youtube_dl
 bs4
 wget
 python-dotenv
+feedparser

sample_config.env

@@ -31,3 +31,5 @@ ARQ_API_KEY=your_api_key # Get it from @ARQRobot
 LOG_MENTIONS=True # Make It False If You Don't Want It
 
 SPAM_CHECK_EXCEPTION_GROUPS=-16628896 # Group where spam protection will be ignored , Seperated by space
+
+RSS_DELAY=300 # In seconds

sample_config.py

@@ -36,6 +36,7 @@ if HEROKU:
             "SPAM_CHECK_EXCEPTION_GROUPS", ""
         ).split()
     )
+    RSS_DELAY = int(environ.get("RSS_DELAY", None))
 else:
     BOT_TOKEN = "467677575:YZfaakjwd545dfg-N6JStihhuw5gQeZHntc"
     API_ID = 123456
@@ -58,3 +59,4 @@ else:
     SPAM_CHECK_EXCEPTION_GROUPS = (
         []
     )  # Leave it empty if you don't know about it
+    RSS_DELAY = 300  # In seconds

wbb/core/filters.py

@@ -24,7 +24,8 @@ SOFTWARE.
 from pyrogram import filters as filters_
 from pyrogram.types import Message
 
-from wbb import USERBOT_ID as OWNER_ID, SUDOERS
+from wbb import SUDOERS
+from wbb import USERBOT_ID as OWNER_ID
 from wbb.modules.trust import get_spam_data
 from wbb.utils.functions import get_urls_from_text
 

wbb/modules/admin.py

@@ -173,7 +173,9 @@ async def purgeFunc(client, message: Message):
 
 
 @app.on_message(
-    filters.command(["kick", "dkick"]) & ~filters.edited & ~filters.private
+    filters.command(["kick", "dkick"])
+    & ~filters.edited
+    & ~filters.private
 )
 @adminsOnly("can_restrict_members")
 async def kickFunc(_, message: Message):
@@ -209,7 +211,9 @@ async def kickFunc(_, message: Message):
 
 
 @app.on_message(
-    filters.command(["ban", "dban", "tban"]) & ~filters.edited & ~filters.private
+    filters.command(["ban", "dban", "tban"])
+    & ~filters.edited
+    & ~filters.private
 )
 @adminsOnly("can_restrict_members")
 async def banFunc(_, message: Message):
@@ -477,7 +481,9 @@ async def ban_deleted_accounts(_, message: Message):
 
 
 @app.on_message(
-    filters.command(["warn", "dwarn"]) & ~filters.edited & ~filters.private
+    filters.command(["warn", "dwarn"])
+    & ~filters.edited
+    & ~filters.private
 )
 @adminsOnly("can_restrict_members")
 async def warn_user(_, message: Message):
@@ -514,8 +520,8 @@ async def warn_user(_, message: Message):
     if warns >= 2:
         await message.chat.kick_member(user_id)
         await message.reply_text(
-                    f"Number of warns of {mention} exceeded, BANNED!"
-                )
+            f"Number of warns of {mention} exceeded, BANNED!"
+        )
         await remove_warns(chat_id, await int_to_alpha(user_id))
     else:
         warn = {"warns": warns + 1}

wbb/modules/rss.py

@@ -0,0 +1,101 @@
+from asyncio import get_event_loop, sleep
+from time import time
+
+from pyrogram import filters
+from pyrogram.types import Message
+
+from wbb import RSS_DELAY, app
+from wbb.core.decorators.errors import capture_err
+from wbb.utils.dbfunctions import (add_rss_feed, get_rss_feeds,
+                                   is_rss_active, remove_rss_feed,
+                                   update_rss_feed)
+from wbb.utils.functions import (get_http_status_code,
+                                 get_urls_from_text)
+from wbb.utils.rss import Feed
+
+__MODULE__ = "RSS"
+__HELP__ = f"""
+/add_feed [URL] - Add a feed to chat
+/rm_feed - Remove feed from chat
+
+**Note:** 
+    - This will check for updates every {RSS_DELAY//60} minutes.
+    - You can only add one feed per chat.
+    - Currently RSS and ATOM feeds are supported.
+"""
+
+
+async def rss_worker():
+    print("[INFO]: RSS WORKER STARTED")
+    while True:
+        t1 = time()
+        feeds = await get_rss_feeds()
+        if not feeds:
+            await sleep(RSS_DELAY)
+            continue
+        for _feed in feeds:
+            chat = _feed["chat_id"]
+            url = _feed["url"]
+            last_title = _feed.get("last_title")
+            feed = Feed(url)
+            if feed.title == last_title:
+                continue
+            await app.send_message(
+                chat, feed.parsed(), disable_web_page_preview=True
+            )
+            await update_rss_feed(chat, feed.title)
+        t2 = time()
+        if (t2 - t1) >= RSS_DELAY:
+            continue
+        await sleep(RSS_DELAY - (t2 - t1))
+
+
+loop = get_event_loop()
+loop.create_task(rss_worker())
+
+
+@app.on_message(filters.command("add_feed"))
+async def add_feed_func(_, m: Message):
+    if len(m.command) != 2:
+        return await m.reply("Read 'RSS' section in help menu.")
+    url = m.text.split(None, 1)[1].strip()
+
+    if not url:
+        return await m.reply("[ERROR]: Invalid Argument")
+
+    urls = get_urls_from_text(url)
+    if not urls:
+        return await m.reply("[ERROR]: Invalid URL")
+
+    url = urls[0]
+    status = await get_http_status_code(url)
+    if status != 200:
+        return await m.reply("[ERROR]: Invalid Url")
+
+    ns = "[ERROR]: This feed isn't supported."
+    try:
+        feed = Feed(url)
+    except Exception:
+        return await m.reply(ns)
+    if not feed:
+        return await m.reply(ns)
+
+    chat_id = m.chat.id
+    if await is_rss_active(chat_id):
+        return await m.reply(
+            "[ERROR]: You already have an RSS feed enabled."
+        )
+    try:
+        await m.reply(feed.parsed(), disable_web_page_preview=True)
+    except Exception as e:
+        return await m.reply(str(e))
+    await add_rss_feed(chat_id, feed.url, feed.title)
+
+
+@app.on_message(filters.command("rm_feed"))
+async def rm_feed_func(_, m: Message):
+    if await is_rss_active(m.chat.id):
+        await remove_rss_feed(m.chat.id)
+        await m.reply("Removed RSS Feed")
+    else:
+        await m.reply("There are no active RSS Feeds in this chat.")

wbb/utils/dbfunctions.py

@@ -55,6 +55,7 @@ restart_stagedb = db.restart_stage
 trustdb = db.trust
 flood_toggle_db = db.flood_toggle
 spam_toggle_db = db.spam_toggle
+rssdb = db.rss
 
 """ Notes functions """
 
@@ -845,3 +846,47 @@ async def flood_off(chat_id: int):
     if not is_flood:
         return
     return await flood_toggle_db.insert_one({"chat_id": chat_id})
+
+
+""" RSS DB """
+
+
+async def add_rss_feed(chat_id: int, url: str, last_title: str):
+    return await rssdb.update_one(
+        {"chat_id": chat_id},
+        {"$set": {"url": url, "last_title": last_title}},
+        upsert=True,
+    )
+
+
+async def remove_rss_feed(chat_id: int):
+    return await rssdb.delete_one({"chat_id": chat_id})
+
+
+async def update_rss_feed(chat_id: int, last_title: str):
+    return await rssdb.update_one(
+        {"chat_id": chat_id},
+        {"$set": {"last_title": last_title}},
+        upsert=True,
+    )
+
+
+async def is_rss_active(chat_id: int) -> bool:
+    return await rssdb.find_one({"chat_id": chat_id})
+
+
+async def get_rss_feeds() -> list:
+    feeds = rssdb.find({"chat_id": {"$exists": 1}})
+    feeds = await feeds.to_list(length=10000000)
+    if not feeds:
+        return
+    data = []
+    for feed in feeds:
+        data.append(
+            dict(
+                chat_id=feed["chat_id"],
+                url=feed["url"],
+                last_title=feed["last_title"],
+            )
+        )
+    return data

wbb/utils/http.py

@@ -69,3 +69,11 @@ async def multipost(url: str, times: int, *args, **kwargs):
     return await gather(
         *[post(url, *args, **kwargs) for _ in range(times)]
     )
+
+
+async def resp_get(url: str, *args, **kwargs):
+    return await session.get(url, *args, **kwargs)
+
+
+async def resp_post(url: str, *args, **kwargs):
+    return await session.post(url, *args, **kwargs)

wbb/utils/rss.py

@@ -0,0 +1,41 @@
+from feedparser import parse
+
+
+class Feed:
+    def __init__(self, url: str):
+        self.url = url
+        feed = parse(url)
+        if not feed.get("entries"):
+            return
+        entry = feed["entries"][0]
+        self.title = entry.get("title") or ""
+
+        # We need title to check latest post
+        if not self.title:
+            return
+
+        self.link = entry.get("link") or ""
+        self.published = entry.get("published") or ""
+        self.updated = entry.get("updated") or ""
+        self.author = entry.get("author")
+        self.summary = entry.get("summary") or ""
+
+    def parsed(self):
+        text = ""
+        if self.title:
+            text += f"**Title:** {self.title}\n"
+        if self.author:
+            text += f"**Author:** {self.author}\n"
+        if self.link:
+            text += f"**Link:** {self.link}\n"
+        if self.updated:
+            text += f"**Last Updated:** {self.updated}\n"
+        if self.published:
+            text += f"**Published:** {self.published}\n"
+        if self.summary:
+            if "<div" not in self.summary:
+                text += f"**Summary:** {self.summary}\n"
+
+        if text:
+            text = "\n".join([i.strip() for i in text.splitlines()])
+        return text