Merge pull request #564 from woorst/mime_parser_streamja

Add mime parser: streamja.com

rtv/mime_parsers.py

@@ -67,6 +67,33 @@ class OpenGraphMIMEParser(BaseMIMEParser):
         return url, None
 
 
+class VideoTagMIMEParser(BaseMIMEParser):
+    """
+    <video width="320" height="240" controls>
+        <source src="movie.mp4" res="HD" type="video/mp4">
+        <source src="movie.mp4" res="SD" type="video/mp4">
+        <source src="movie.ogg" type="video/ogg">
+    </video>
+    """
+    pattern = re.compile(r'.*$')
+
+    @staticmethod
+    def get_mimetype(url):
+        page = requests.get(url)
+        soup = BeautifulSoup(page.content, 'html.parser')
+
+        # TODO: Handle pages with multiple videos
+        video = soup.find('video')
+        if video:
+            source = video.find('source', attr={'res': 'HD'})
+            source = source or video.find('source', attr={'type': 'video/mp4'})
+            source = source or video.find('source')
+        if source:
+            return source.get('src'), source.get('type')
+        else:
+            return url, None
+
+
 class GfycatMIMEParser(BaseMIMEParser):
     """
     Gfycat provides a primitive json api to generate image links. URLs can be
@@ -458,6 +485,13 @@ class FlickrMIMEParser(OpenGraphMIMEParser):
     pattern = re.compile(r'https?://(www\.)?flickr\.com/photos/[^/]+/[^/]+/?$')
 
 
+class StreamjaMIMEParser(VideoTagMIMEParser):
+    """
+    Embedded HTML5 video element
+    """
+    pattern = re.compile(r'https?://(www\.)?streamja\.com/[^/]+/?$')
+
+
 class WorldStarHipHopMIMEParser(BaseMIMEParser):
     """
     <video>
@@ -492,6 +526,7 @@ class WorldStarHipHopMIMEParser(BaseMIMEParser):
 
 # Parsers should be listed in the order they will be checked
 parsers = [
+    StreamjaMIMEParser,
     ClippitUserMIMEParser,
     OddshotMIMEParser,
     StreamableMIMEParser,

tests/cassettes/test_parser[streamja].yaml

@@ -0,0 +1,138 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept: ['*/*']
+      Accept-Encoding: ['gzip, deflate']
+      Connection: [keep-alive]
+      User-Agent: [python-requests/2.14.2]
+    method: GET
+    uri: https://streamja.com/6WMb
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAA+xba3ObTLL+fn6FwlvrV2yQDAglsWw5RxaKg5cBy0Jx5K2tLQRjNIiLlou52P7v
+        p3pAEvIl++6nra2zlXIMMz0zfXmme7oHn32Q9bGxuJ60VonvnZ/V/2PTPj/zcWK2rJUZxTgZMmly
+        3/nCHJ+fJSTx8PksibDpu2ar05oRf+Ph1gOxcdiKV2ZEAufsuCKrJglMHw+ZB4KzTRglTMsKgwQH
+        yZDJiJ2shjZ+IBbu0BeOBCQhpteJLdPDQ6HLc76ZEz/1Gy1pjCP6ai49POSBK48E69YqwvdDZpUk
+        m3hwfHwfBkncdcLQ8bC5IXHXCv1jK46/3ps+8YrhTbgMk3DQ43lO4nmmFWFvyMRJ4eF4hXHCHEwK
+        A49pZ9eK47eI6UsrKTZ4yCQ4T2AEc/4bVcvfQWKTBDh6pFIOBJ7/02n4gKN7L8wGK2LbODjdhDFJ
+        SBgMIuyZCXnAp8swsnHUiUybpPFA2uTPL+er1P74gKOEWKbXMT3iBAOf2LaHT8ONaZGkGPD1sHEY
+        JFHoxY/LMO/EpCSBM6iXWIb56dK01k4UpoE98EiAzajjwMo4SNqRszTbPEf/sVzjrSt+ZtnThlAg
+        es2Fh++TvVDmMg69NMGvxTZtGzjp9Td5i2+J/Cbfc36aRGZQz1A3trpCP25hM8YdEnTCNDmNiLNK
+        BvwpLDjgT5dhkoQ+SO2nCX6890IzqZhZYUopSJt8yzM80nHi501+aqVRHEaDTUiCBEdvWWSvozTy
+        2raZmAPimw4+jh+cj7nvnabJ/ZfTpRnjTxJ3LZ9Iy9vMsf0fhSV6D0uXJ2gmZQq58Ja+9nB36aV3
+        JU/Mn9qDajiSZqwTNFOOr53wizLjE2Vie0v/R7H4efNguaEzvb15WPhzZ3abx/ZPjbf8b/yypzjI
+        WKRoLGXq+ML4sbacm5+rbNm74pVvF7F9ayWzW8lRSdX3Y8dL6GhEytD4YmzfevHdeJT9ZTZy1Bn/
+        cexmq5vJifzjmze9mV2Utm85828/xsbEk5WxkqjFyU/EoweVv5nf1GuimZSrxck3gyiO4q94+/vo
+        k1qc9OyeldolSpe9q0Dl7WLx8yI0b7VSFbTbm+JLrrqTh5tvNxO1p4l35SRX/Rv+bqx8HLs5XRdf
+        CvEyQCfvzlkqGZJHD1bvLlDIhbS8zVOr3EjLSy81Qbffb3hLDh/Unt2zi34PFf0Hy7cekLHu67OT
+        iq5QXtooV8jF5k7myVT8li1+fIF36XqmZNZ3hyjf1yeKe5HhseLYore2L50TxZ2ItO9y5Zm3dmjL
+        PEHGXUXje57NXz1gaBuPMkVe5Ggm9XV54mjuJFfdtYRmimP1bvrLy/mJ4v9IF7dX8d1MIItbLbrr
+        XT3Yt/217vc9u4Dxo8a4kYiMSaobTq4XCtWXHmjZ4lbzrmdXmeX/KO9+Xol3M6rT4NoJr64vrY9j
+        Z311/f1iZV86DshpGIqkjSVBG2eiNpYyTUSJPpYKrXQSVEg8cicJMqxUlyeFakxSzZjkqqHwqquU
+        2izjVdfpa5MsR+481WVUqLKSaWOp0I0rBblKitx5Pi3XqW5MxH2fEqNxViBjmiJ3KqjyKEauIqru
+        nNe+L3LNHaWaO8qV8chRLlGmwtqu0tdnWV9117lGsgIRqdCMSbxdQ5UVoBHQ3KG8oPImRgYSVHdU
+        6uMsRyW0Kf1FCWvOe6oMvy1BlYE3p6/Ki1QzgP+JoLoWjAF+SkRGjjK+cBGRJFQ6wDf0SxrhM8ov
+        ySTVnfa1gs9Vd9FHPpJUFwloxpeqOy3QLMtRIfU0Y54gqr85r8qTUnUtQZvxBQLdy99udXmaaq4i
+        TUulr4+lUpPXsVbOoa1UZaUP6+vGJNZcK4U+5Do8rK/KC151Rz1MFOfO9+KlzBOFv7u8Wd9d3hDl
+        Aew+NvJs8fMmVC6nJ8qaF6jfoPZAmUakAhl3LhrzEiqkUnPniSavU82YlqoxEfSxxGsGSpAB9lEE
+        1Zj00QzobFcFLFC78ICDUp/xhTaWeGQsYjSWSl1GCXLXKXKVmMrurvu1TRNtLEmaYdF25DqUXnOn
+        MXKnKSoXYAfQp4hIVugzqUDylUtxIY8EVVYE1XV6iGR9fSaVyEUx6E83UK7K677qogIVWa4ZQL8Q
+        F+Uo1gATgDUD7GPB2JyuZcwl1VhLwD+0wRgkT3d8VvwBP9NUly1RBb25U4nOX0h9zbAquwJ+ZaWv
+        ukqxKBG856rspJqs8JUsTqERigVek+cxAlkMkAWwMOW1W1SATNosy1QZ9tJaVIF/wynoPKWVoBnF
+        oAO8TcupqAEmXSfWKAYmPX2WNW0rgd40Yx5rlOcJ7JsSyYsUuRPxX8ALD/tfc50tXnId9qQxBdzx
+        +pgvaJusJLpcy20sRKrjgs+RPKJ+YQHPwI+rgO17CDBFbT3lVRnsZJWqAfqb5LW+Mn1c7T3VqHVP
+        2xHIkanuNNfHGeCt2t8zmNOKKd4Mh1+U4KNGfVWeZKo7kfRxVuozSUSuFSNqn7WgyqivuotCKzJB
+        n0mSJn9zwS9pVM+LBBngM9Zg+z71d2Me5sg1A8WaAXvE6R3iZN1/gRMeeKM4ofPB/l9XPLpzscIJ
+        +CInRaUjvcBJqZWowgn1q4AT1EMi4GTNawXgZJEiecQ3caLLVh/8tC7PRcrbBJU6rFtaW0zkmryI
+        NXnSU92FBJig+71EkipPgVdRlxcxKkeAGelfwgn4JtfiVRl8hUNjyU4v1IZrvoodUwEB/y7iNcJX
+        +5tQG4qaq72wN/X7Ne2BvUUN/ArV5S5W9LWZJOhgH3mSIlkBnQoa6BL2nIF41V0IGt3nFmAPfEmp
+        jyVBl6/A9jkCDFPft8hVdw5Yy3VjUcUIYyLQPQf+zQU/Mur/2v5z6v/B/qhcJ1sfeOgnKvv/0k/s
+        7L/1E2vA1db+wtv2R1ts5hXuHFE1ppLqTjJtluV0r5aKSGUqJJHuHcOBuFSATbTSAX/am5aLHGxD
+        4xPoT57TfaSVTqyVIOs01mQFsNTTYO+AfmT7k0IufPM2j69niksxs767VMgXOH98UUULzpoPd6X0
+        F3om+U97Hp/AGfXj2BkO2VYQdiK8wWbSyBIgzcKDOl17/u0+9bzYijAO6rSEpi7v5SVVXvOfnZg4
+        /01M/l2JyaukY54jMhKQoRDlu8bj2xwSBG/pfyPLy3my8L+5pmgXy96P9E7epHc/LefFuM+VHNkn
+        q3exWojzEyW4KO5+ap4V3HkKueiZtze82Uh8zMsfmztxxV/PlFwLLiTFrTeQs/5iXX7jzfHF+np2
+        pSEDguuCn8momJYTHg5uOk1EwLlZUv2eqfJatN01PchVhxELHOKWHg5u1U+ASkT4HIIEPVSWSoLK
+        K1E1FrwpTwr9EtEAp5U0AXmDVoEDh4Hg4OAu4JAIvOSQsGjuzS+dWlMuTXayrVyIBoctzyNYI6aJ
+        T6lIKk04UGGVb8jiQyKzEjX3hgamHd/bsXI1thqPIDlrzMGXyFv0NUIPAGDDGBx6NQ6SlWouOHBv
+        ef1VoD9MGKcSHJJ0eQUJQK/WU1bJsuWBr+2xELWJUyL34g/QoqYt6p+rEBmLbFE2xspvyBosEt1Y
+        gO1kzZiATjJEqrk0eb0dW+iyU8urSPr3kCiXd5vlZXaiEK2Sd70gain95cXBhtoflQ4/kxe8TXhR
+        owfgLU87DCU1hgBzBsWXrNQ6H4E82R/CsO/kmnvhNjBTYbNpdwMVENznpZJVwbjiT5UnfZhDl3+B
+        1f9PgXfjmcVFmiRh8GiTGN4GQRj8t57337D5i3peoX+/kBRyEd7deoH5fdpsE83bH72pf0LngHoc
+        oq57JOrjUUm3qLwAutL+7sV3Bk/ubvurhZ976u3VaiEmgeWfCEt/+mnp/+jR8c1xMnWBmeY6vw65
+        jXB6V1Ty/ew5JRK+CBrp388v8xWGuh/U+uqa3wuXVugGgtyhqMLsvJyWCk9dCeR+UPcrIV+al6q7
+        yCHHh3qbbixoGNEgT5eVnMo9QZCXlHX9kEeGk1Q1uTXkXiKC+pEMeRrkYwuokZSQo6Fy6tT5CU9z
+        3VnW02ZSphs0T4MaDtRCtvUDGA95o6jNMshXeI1HJYK8x1V6qlzxjso15PYizZMh7zJoPQJyxFJ1
+        RxkiWQ9yPdQLHcNQcp3m+VBzWkA9KUOzEeVpUfDAd6bRWgHU5dYQckqN1uCmVQg2RpA79dBMEiC3
+        16BmY1jltIQ8bS7SvB9qBOUiV2XIOycimmUSIpALzmOd1j4hT3WgvrTVRbVuOUnqdUEXIq2xjmmY
+        yRHYx7AgN+3p44zXZlBb+CajcgQ1kX5V94JQMQK99PSZ1KdjXJTTGlABddc16JTm6bTWIX/7pEz4
+        AoG+jDnUpAQ0lkTNuIpVyCtdJdfGPE/rGPLa2dXwConXDVrPKbVymlTjEc1Hddnqgc5qnnpQM9Vv
+        ad0GcmLI03OthKONlEFdVYf14DhCoL4CGFBSZKzzKu9diPoM5nAEdJtJoHNdhjqlBXYolapOJCOX
+        YrLQwfaAQRlyYqUAX13l/3OoBxe0fvxa1l8d766uxydVDb36Xeew+X/gc0Z97LUT/jKEfuY3eQv+
+        a/bs8k8LH+aju1u+JNw0ruK2V3Pbm7ptwgs3ho3LwxdZ7j3xEhwN7CjcdOKVaYdZG5iBn94mbzUv
+        JbtCn2V314bdL/3mxaHpeS2R5/34tOyQwMb54OTkpHkmGKzgUvJxO1zYXrN6oQlXk4/vibcVZ3uF
+        KcAVpvBlk28FpJeZlYDwuJ03xngN8+6vOesr8cHvv5++fTn7Ngt9/k+Vavs7RdIlm8v7ZuSQoAPk
+        HaHRQMdVLYf3zDBX1TIQN3krDj1iV8oW+31u+8N3++x2YBJuOlbohdHgt/v7+1MzIL5JmT0Qt9U9
+        iVvVzXKLBPdw048bZurYGI5lQrcfP//vGhf3kenjuHWosSR8pCPuw8gfRGFiJrjd+8Tb2GGfn3/b
+        RKET4Tim9+OPL9BU6wQKK1t70RvnSoGf+F3BhT6+oe89oHYstLrCl8O76AYEX/BTg2zPPv2gYdEW
+        utIeucJ+UGs/fBnmzWvsxo5pCPXqY4Hmhq2s88qIsDTFG91dA77Fv9xZXZHdqYs/YM4zi79vX3eH
+        6qUXWuuaQf6Au5e8UKS85rmhZTpLqytsQfN8Vn2BcX52XH2hsgztovpaBUctYg+Z6pE5P7PJQ8vy
+        zDgeMl7ohMz5mbn9kIM5PyO+06pYGzIiz7TiyBoyx8R3joG4uwkc5vzs2Dw/O7bJw8FkPg7SymUw
+        rTCwPGKth4yNcJC2WYay0KA4P4s3ZnB+dlz9ev+lWsbcLlJJoZJgXU2J/SW2mRb9ombITOCt9QN2
+        BdOKV2GUAM2nW7RkahF/a/BmhZti7JFNO1mRmGslUYrZ0wgnaRS07k0vxqfMOZ2RytuQ9J54WAnu
+        wxbdEzO8Yc4FrseL3Oee2PpBcBbXbB9X7FaDgRWqjqbG6Q6OmXPKc0wX2vV5oUMC5lwNnZYSHHbF
+        xAnSDXM+I07Qmm8OOxMc+TFzfh2RB9MqWkctAxreMRr9GuYQFSRAtLUVhGjbHVsR2SQtMy4Cq8bE
+        8cZ0sGmL9WdEcRHYxKKujX5NVPUeu/GxacfLoiLqujHAp5rtvHX2odNpqVRHy9CM7Fanc946I0G8
+        ZaUxlGlRgA+Z7XYiAUC/09xVn8VGfDlp+PM61gpv+nymBaltx7Q7lkfot1eW2dmky84nXuhJ/JeT
+        T72TLyeieLInjL0wGTJ9QZT6/Z74SQKhSBCft862orUbrLeGrYwEdph1m41PT62//o3tbtJ41X58
+        Zk9bO700jQGi4mi8/aaJ2WPpxddOzPlZ9ZnZJowTHO2/9bLswDW7UTcLI8+OY68b4OSYHH/KjmFj
+        dN2Nw7S8MNy0zDQJYbnd/gcvyrSqD9Gql/OzOEwjC1cQ+OUK/kbareFvJKb+/IvyCJ2gMvqyl2h/
+        6GCaUN0LW585XimhjoC7UW+M3X7f1RgLX0C9WuggLDGv299oghBUO7SdGH9v0B94srcY3N98MG+R
+        vf6/3o2ND+pc88GsWpnz2L2moGlTJ8ae3qeBBduytevA7OODGbUSLhjaoZX6OEi6Dk4mHobH+KIw
+        TEczfdyudMewf+X/xoVv0V4Uit1+BUWWi/8J8c4cLBcN2/9k4q15We5dwp26/wBNZVqW5cj7TB7a
+        kOXMX5PWqGW59H06CjeW896naOCA5axhkHoeZw+35muzj9YwxolBfBymSbvRHnerzy/rk9KQ58K6
+        pTrnDRko6DHPnNjnefaZ8/eTYvbROjqyPGxG24ktlns5ofBqQhvfm6mXMJzdZp+5YvgIR9ZBgycA
+        GB5CaD3lh8Nh0I2waRczOJV+DbqmbU8ecJCoJE5wgKM2A3sb2/BNLDhZhmtMhbskiBPT82r4ss/c
+        B4EdNKc8HwpHR6/oONxdmYHt4co+12BMHNN2iqyZFaXLJQkcmPJgbFOS4JUyPoBE+GvbrHvqkDRk
+        aDBiOPMN+eih40CsoguDrs00xm2WezUXNRlXdONVmH0nNt5umVp8dtAuuklkWmvg+LqGapt9cwTL
+        pX+IJQEM9RB6qY+/trdPw26fS1/ooPuZHXT7b1Lzr4kldsDvaY+O9sTCK2KhEo/z/hDDQTfDyzVJ
+        bvA/Uhwn31LPm9Ed9PXdnnaFnNf00ZuUfli+muLo6O322jTP3EsLNOEUdsPAD9MY++EDbu5uH4bv
+        ej1sHna/2uTPz9wb6D5E7rs6hI25x1/F9e71cI4NNNlPT0EXBza2wdT0gdrRSqMIB9QpDXmWC+ic
+        FIVvYJMdtOvpKpwm4cYAKhI4LwifuVfDm0x92MU0TDdSurHNBL/YCEnTW2JOANcHKn5N3Jya1Gqu
+        jj0HAh4HXTuN6BH3zwLPf2T+xDxz7wnRnPPAvSbsM3fofZqk0Rsmo3iwwyx4Af0/osl9OHwTdRi0
+        F+NEjkxnNwh34fD+k31+NTjdHAx9o7sKWm+tSXvoYgcc7hZ75j5AbHrRP2iuRw8tQ2Qmq65v5m2e
+        qx5J0Ba4Nu5QTN+TwL4O45/tiGWPo254fx/j5BaMybKnh3hN/rw3KPcOiH4NaMpuU3f/Znafue18
+        B4zch1G7YgbXK6hQK8G712sTVjllk4+HFHU2nDw/nx6PAjsKif2U4aU+eyLXqzDAT+TatJ/IdWg/
+        XXimtb7AUVQ8KRMULomHn/QNjswWIgE5Jt0Ex0k7MB+IYyZh1IU/VBk5OEhYcCgvI2zc3VCWtNDG
+        3YgCaLwint2OwSQxTkZJEpFlmuA2Y23PkhzDsOzgw6tDyHHNZc1xaB+/z8zR0Yc6V0Oz6g95uGT4
+        BmGXFkv1+zYzXkWhjxn2vCNwwa9JZ+a9GRFKulPt0VEAjnQIMS94esLPbfZr0YXTVJuGePqEhx+o
+        c6000ZB9m7MxLPu884l10eXx3YNmVYZguzTHBD/TTULH8XCbCTf08Pn+IXZfwnl/+J6TXYkFc0ll
+        jZALuIjzOJOzOcJZp8Ew+fo7zYTq9P51Ae+g5Lj7qKsuvG1T+/6nbk/8/KdTKGHR6uhhihrXf5NF
+        SxO0aHT8+8d2XT4aDtveEHeJzX7FIHFDv3URiWEHuEZkrY4Xb8RmP/7OtOjKVeFuyPCHufOLtNr0
+        vDDbn/jfF58+vy650pIDXa/OCn8fvC0t8/HXrL8UhGs3Mj0rwmaC6842zTXNCJsMy3YfTC/Fw4Az
+        h7jevvs65hjKmJy976EqqVrJrpUWOzkLVB8EOPpuIJXb9cWxgfNkuDOR95WpiqOt38TPJv7EnzID
+        5nXltO6ri6eHjY3iKsM1Fh0y43BDsM1AlnNgfkgwwsArqGvZ50BVogt/5BYRK2F2HVsrDZmtmfZ9
+        UJ0eMp2Tk5OTTb5vhr/Em5ESDxlB3CTMKeyRePhBOG1HB4niDHu49mpsNzIDB4/DNEjO+aOjdjyM
+        gOQGWkdJm2cbGxjqwF1zs8GBXTnQkMpBZ2vvH0kYzBIzSqrUcds0Cexh0PVw4CQrypkPnCVR8ejv
+        mcM5tsah75uB3aY1VUj/30xTX9n2lfmYj+ZH5tB4zEf7I3O6fSYfmRe2s565Hu6xz5aZWCsIds+1
+        Zz1UQTOGhCwXHx21o6079TyqOgj39PhF39oxy3L+8z6aYC7hTC7gLC7kYhDmktbxRoHpFQmxYn3p
+        YisZWhz+q/W3Ifz39NQQvg0t3X8Mq19PT7vanxk5lM+YfaZDu95Q+HOAs5ZsJpgLhy83oQnVlXfq
+        NWZVqenSOm2VtTe9YJZldbW2Y27Zpj5i/+bGzGHwJUGMo+QC34cRboPkz+0qQO4wxjF15YnjOcYx
+        GZZzzDZTMc1wzHzUEYQe3/si8kJHYDgas2qiGAc2wzFwAoQ/gWXY030R9Li6wjimf3f7P/8HAAD/
+        /wMAARKVwo47AAA=
+    headers:
+      CF-RAY: [42e3293bbc2cab90-YYZ]
+      Connection: [keep-alive]
+      Content-Encoding: [gzip]
+      Content-Type: [text/html; charset=UTF-8]
+      Date: ['Thu, 21 Jun 2018 02:49:17 GMT']
+      Expect-CT: ['max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"']
+      Server: [cloudflare]
+      Set-Cookie: ['__cfduid=db88cc12581eacd4f5fca60cc7d507b631529549357; expires=Fri,
+          21-Jun-19 02:49:17 GMT; path=/; domain=.streamja.com; HttpOnly']
+      Strict-Transport-Security: [max-age=15768000]
+      Vary: [Accept-Encoding]
+    status: {code: 200, message: OK}
+version: 1

tests/test_mime_parsers.py

@@ -134,6 +134,10 @@ URLS = OrderedDict([
         'http://www.worldstarhiphop.com/videos/video.php?v=wshhJ6bVdAv0iMrNGFZG',
         'http://www.youtube.com/embed/Bze53qwHS8o?autoplay=1',
         'video/x-youtube')),
+    ('streamja', (
+        'https://streamja.com/6WMb',
+        'https://cdnja.r.worldssl.net/mp4/6w/6WMb.mp4',
+        'video/mp4')),
 ])