From 452ccfd567f79126e108f69bb7ebca07b5993bdd Mon Sep 17 00:00:00 2001 From: Miguel Ángel Moreno Date: Mon, 21 Nov 2022 17:55:28 +0100 Subject: feat: Initial commit --- src/backend/tau/api/channel.clj | 52 ++++++++++++++++++++++++++ src/backend/tau/api/comment.clj | 47 ++++++++++++++++++++++++ src/backend/tau/api/kiosk.clj | 47 ++++++++++++++++++++++++ src/backend/tau/api/playlist.clj | 51 ++++++++++++++++++++++++++ src/backend/tau/api/search.clj | 49 +++++++++++++++++++++++++ src/backend/tau/api/service.clj | 24 ++++++++++++ src/backend/tau/api/stream.clj | 66 +++++++++++++++++++++++++++++++++ src/backend/tau/core.clj | 11 ++++++ src/backend/tau/downloader_impl.clj | 64 ++++++++++++++++++++++++++++++++ src/backend/tau/services/http.clj | 73 +++++++++++++++++++++++++++++++++++++ src/backend/tau/utils.clj | 1 + 11 files changed, 485 insertions(+) create mode 100644 src/backend/tau/api/channel.clj create mode 100644 src/backend/tau/api/comment.clj create mode 100644 src/backend/tau/api/kiosk.clj create mode 100644 src/backend/tau/api/playlist.clj create mode 100644 src/backend/tau/api/search.clj create mode 100644 src/backend/tau/api/service.clj create mode 100644 src/backend/tau/api/stream.clj create mode 100644 src/backend/tau/core.clj create mode 100644 src/backend/tau/downloader_impl.clj create mode 100644 src/backend/tau/services/http.clj create mode 100644 src/backend/tau/utils.clj (limited to 'src/backend/tau') diff --git a/src/backend/tau/api/channel.clj b/src/backend/tau/api/channel.clj new file mode 100644 index 0000000..26d3b34 --- /dev/null +++ b/src/backend/tau/api/channel.clj @@ -0,0 +1,52 @@ +(ns tau.api.channel + (:require + [tau.api.stream :as stream] + [clojure.java.data :as j] + [ring.util.codec :refer [url-decode]]) + (:import + org.schabi.newpipe.extractor.channel.ChannelInfo + org.schabi.newpipe.extractor.NewPipe + org.schabi.newpipe.extractor.Page)) + +(defrecord Channel + [id name description verified? banner avatar + subscriber-count donation-links next-page + related-streams]) + +(defrecord ChannelResult + [name description verified? thumbnail-url url + subscriber-count stream-count]) + +(defrecord ChannelPage + [next-page related-streams]) + +(defn get-channel-result + [channel] + (map->ChannelResult + {:name (.getName channel) + :thumbnail-url (.getThumbnailUrl channel) + :url (.getUrl channel) + :description (.getDescription channel) + :subscriber-count (.getSubscriberCount channel) + :stream-count (.getStreamCount channel) + :verified? (.isVerified channel)})) + +(defn get-channel-info + ([url] + (let [info (ChannelInfo/getInfo (url-decode url))] + (map->Channel + {:id (.getId info) + :name (.getName info) + :verified? (.isVerified info) + :banner (.getBannerUrl info) + :avatar (.getAvatarUrl info) + :subscriber-count (.getSubscriberCount info) + :donation-links (.getDonationLinks info) + :next-page (j/from-java (.getNextPage info)) + :related-streams (map #(stream/get-stream-result %) (.getRelatedItems info))}))) + ([url page-url] + (let [service (NewPipe/getServiceByUrl (url-decode url)) + info (ChannelInfo/getMoreItems service url (Page. (url-decode page-url)))] + (map->ChannelPage + {:related-streams (map #(stream/get-stream-result %) (.getItems info)) + :next-page (j/from-java (.getNextPage info))})))) diff --git a/src/backend/tau/api/comment.clj b/src/backend/tau/api/comment.clj new file mode 100644 index 0000000..18881cb --- /dev/null +++ b/src/backend/tau/api/comment.clj @@ -0,0 +1,47 @@ +(ns tau.api.comment + (:require + [clojure.java.data :as j] + [ring.util.codec :refer [url-decode]]) + (:import + org.schabi.newpipe.extractor.NewPipe + org.schabi.newpipe.extractor.Page + org.schabi.newpipe.extractor.comments.CommentsInfoItem + org.schabi.newpipe.extractor.comments.CommentsInfo)) + +(defrecord CommentsPage + [next-page disabled? comments]) + +(defrecord Comment + [id text upload-name upload-avatar upload-date upload-url + upload-verified? like-count hearted-by-upload? pinned? replies]) + +(defn get-comment-result + [comment] + (map->Comment + {:id (.getCommentId comment) + :text (.getCommentText comment) + :upload-name (.getUploaderName comment) + :upload-avatar (.getUploaderAvatarUrl comment) + :upload-date (.getTextualUploadDate comment) + :upload-url (.getUploaderUrl comment) + :upload-verified? (.isUploaderVerified comment) + :like-count (.getLikeCount comment) + :hearted-by-upload? (.isHeartedByUploader comment) + :pinned? (.isPinned comment) + :replies (when (.getReplies comment) + (j/from-java (.getReplies comment)))})) + +(defn get-comments-info + ([url] + (let [info (CommentsInfo/getInfo (url-decode url))] + (map->CommentsPage + {:comments (map #(get-comment-result %) (.getRelatedItems info)) + :next-page (j/from-java (.getNextPage info)) + :disabled? (.isCommentsDisabled info)}))) + ([url page-url] + (let [service (NewPipe/getServiceByUrl (url-decode url)) + info (CommentsInfo/getMoreItems service url (Page. (url-decode page-url)))] + (map->CommentsPage + {:comments (map #(get-comment-result %) (.getItems info)) + :next-page (j/from-java (.getNextPage info)) + :disabled? false})))) diff --git a/src/backend/tau/api/kiosk.clj b/src/backend/tau/api/kiosk.clj new file mode 100644 index 0000000..81e0030 --- /dev/null +++ b/src/backend/tau/api/kiosk.clj @@ -0,0 +1,47 @@ +(ns tau.api.kiosk + (:require + [clojure.java.data :as j] + [tau.api.stream :as stream] + [ring.util.codec :refer [url-decode]]) + (:import + org.schabi.newpipe.extractor.StreamingService + org.schabi.newpipe.extractor.Page + org.schabi.newpipe.extractor.kiosk.KioskInfo + org.schabi.newpipe.extractor.NewPipe)) + +(defrecord KioskList + [default-kiosk available-kiosks]) + +(defrecord Kiosk + [id url next-page related-streams]) + +(defrecord KioskPage + [next-page related-streams]) + +(defn get-kiosk-info + ([kiosk-id service-id] + (let [service (NewPipe/getService service-id) + extractor (.getExtractorById (.getKioskList service) kiosk-id nil) + info (KioskInfo/getInfo extractor)] + (map->Kiosk + {:id (.getId info) + :url (.getUrl info) + :next-page (j/from-java (.getNextPage info)) + :related-streams (map #(stream/get-stream-result %) (.getRelatedItems info))}))) + ([kiosk-id service-id page-url] + (let [service (NewPipe/getService service-id) + extractor (.getExtractorById (.getKioskList service) kiosk-id nil) + url (url-decode page-url) + kiosk-info (KioskInfo/getInfo extractor) + info (KioskInfo/getMoreItems service (.getUrl kiosk-info) (Page. url))] + (map->KioskPage + {:next-page (j/from-java (.getNextPage info)) + :related-streams (map #(stream/get-stream-result %) (.getItems info))})))) + +(defn get-kiosk-list-info + [service-id] + (let [service (NewPipe/getService service-id) + kiosks (.getKioskList service)] + (map->KioskList + {:default-kiosk (.getDefaultKioskId kiosks) + :available-kiosks (.getAvailableKiosks kiosks)}))) diff --git a/src/backend/tau/api/playlist.clj b/src/backend/tau/api/playlist.clj new file mode 100644 index 0000000..ccc0d6a --- /dev/null +++ b/src/backend/tau/api/playlist.clj @@ -0,0 +1,51 @@ +(ns tau.api.playlist + (:require + [clojure.java.data :as j] + [tau.api.stream :as stream] + [ring.util.codec :refer [url-decode]]) + (:import + org.schabi.newpipe.extractor.playlist.PlaylistInfo + org.schabi.newpipe.extractor.Page + org.schabi.newpipe.extractor.NewPipe)) + +(defrecord Playlist + [id name playlist-type thumbnail-url uploader-name uploader-url + uploader-avatar banner-url next-page stream-count related-streams]) + +(defrecord PlaylistResult + [name thumbnail-url url upload-author stream-count]) + +(defrecord PlaylistPage + [next-page related-streams]) + +(defn get-playlist-result + [playlist] + (map->PlaylistResult + {:name (.getName playlist) + :thumbnail-url (.getThumbnailUrl playlist) + :url (.getUrl playlist) + :upload-author (.getUploaderName playlist) + :stream-count (.getStreamCount playlist)})) + +(defn get-playlist-info + ([url] + (let [service (NewPipe/getServiceByUrl (url-decode url)) + info (PlaylistInfo/getInfo service (url-decode url))] + (map->Playlist + {:id (.getId info) + :name (.getName info) + :playlist-type (j/from-java (.getPlaylistType info)) + :thumbnail-url (.getThumbnailUrl info) + :banner-url (.getBannerUrl info) + :uploader-name (.getUploaderName info) + :uploader-url (.getUploaderUrl info) + :uploader-avatar (.getUploaderAvatarUrl info) + :stream-count (.getStreamCount info) + :next-page (j/from-java (.getNextPage info)) + :related-streams (map #(stream/get-stream-result %) (.getRelatedItems info))}))) + ([url page-url] + (let [service (NewPipe/getServiceByUrl (url-decode url)) + info (PlaylistInfo/getMoreItems service url (Page. (url-decode page-url)))] + (map->PlaylistPage + {:next-page (j/from-java (.getNextPage info)) + :related-streams (map #(stream/get-stream-result %) (.getItems info))})))) diff --git a/src/backend/tau/api/search.clj b/src/backend/tau/api/search.clj new file mode 100644 index 0000000..4969f10 --- /dev/null +++ b/src/backend/tau/api/search.clj @@ -0,0 +1,49 @@ +(ns tau.api.search + (:require + [tau.api.stream :as stream] + [tau.api.channel :as channel] + [tau.api.playlist :as playlist] + [clojure.java.data :as j] + [ring.util.codec :refer [url-encode url-decode]]) + (:import + org.schabi.newpipe.extractor.search.SearchInfo + org.schabi.newpipe.extractor.InfoItem + org.schabi.newpipe.extractor.NewPipe + org.schabi.newpipe.extractor.Page)) + +(defrecord SearchResult + [items next-page search-suggestion corrected-search?]) + +(defrecord SearchResultPage + [items next-page]) + +(defn get-search-results + [items] + (map #(case (.name (.getInfoType %)) + "STREAM" (stream/get-stream-result %) + "CHANNEL" (channel/get-channel-result %) + "PLAYLIST" (playlist/get-playlist-result %)) + items)) + +(defn get-search-info + ([service-id query content-filters sort-filter] + (let [service (NewPipe/getService service-id) + query-handler (.. service + (getSearchQHFactory) + (fromQuery query (or content-filters '()) (or sort-filter ""))) + info (SearchInfo/getInfo service query-handler)] + (map->SearchResult + {:items (get-search-results (.getRelatedItems info)) + :next-page (j/from-java (.getNextPage info)) + :search-suggestion (.getSearchSuggestion info) + :corrected-search? (.isCorrectedSearch info)}))) + ([service-id query content-filters sort-filter page-url] + (let [service (NewPipe/getService service-id) + url (url-decode page-url) + query-handler (.. service + (getSearchQHFactory) + (fromQuery query (or content-filters '()) (or sort-filter ""))) + info (SearchInfo/getMoreItems service query-handler (Page. url))] + (map->SearchResultPage + {:items (get-search-results (.getItems info)) + :next-page (j/from-java (.getNextPage info))})))) diff --git a/src/backend/tau/api/service.clj b/src/backend/tau/api/service.clj new file mode 100644 index 0000000..de2e994 --- /dev/null +++ b/src/backend/tau/api/service.clj @@ -0,0 +1,24 @@ +(ns tau.api.service + (:require + [clojure.java.data :as j] + [tau.api.kiosk :as kiosk]) + (:import + org.schabi.newpipe.extractor.NewPipe + org.schabi.newpipe.extractor.kiosk.KioskList + org.schabi.newpipe.extractor.StreamingService)) + +(defrecord Service + [id info base-url kiosk-list]) + +(defn get-service-info + [service] + (map->Service + {:id (.getServiceId service) + :info (j/from-java (.getServiceInfo service)) + :base-url (.getBaseUrl service) + :kiosk-list (map #(kiosk/get-kiosk-info % (.getServiceId service)) + (.getAvailableKiosks (.getKioskList service)))})) + +(defn get-service-list-info + [] + (map #(get-service-info %) (NewPipe/getServices))) diff --git a/src/backend/tau/api/stream.clj b/src/backend/tau/api/stream.clj new file mode 100644 index 0000000..d0a7c81 --- /dev/null +++ b/src/backend/tau/api/stream.clj @@ -0,0 +1,66 @@ +(ns tau.api.stream + (:require + [clojure.java.data :as j] + [ring.util.codec :refer [url-decode]]) + (:import + org.schabi.newpipe.extractor.stream.StreamInfo + org.schabi.newpipe.extractor.NewPipe + org.schabi.newpipe.extractor.localization.DateWrapper + java.time.Instant)) + +(defrecord Stream + [name description upload-date + upload-author upload-url upload-avatar + thumbnail-url service-id duration view-count like-count + dislike-count subscriber-count upload-verified? hls-url + dash-mpd-url category tags audio-streams video-streams + related-streams]) + +(defrecord StreamResult + [name url thumbnail-url upload-author upload-url + upload-avatar upload-date short-description + duration view-count uploaded verified?]) + +(defn get-stream-result + [stream] + (map->StreamResult + {:url (.getUrl stream) + :name (.getName stream) + :thumbnail-url (.getThumbnailUrl stream) + :upload-author (.getUploaderName stream) + :upload-url (.getUploaderUrl stream) + :upload-avatar (.getUploaderAvatarUrl stream) + :upload-date (.getTextualUploadDate stream) + :short-description (.getShortDescription stream) + :duration (.getDuration stream) + :view-count (.getViewCount stream) + :uploaded (if (.getUploadDate stream) + (.. stream (getUploadDate) (offsetDateTime) (toInstant) (toEpochMilli)) + -1) + :verified? (.isUploaderVerified stream)})) + +(defn get-stream-info + [url] + (let [info (StreamInfo/getInfo (url-decode url))] + (map->Stream + {:name (.getName info) + :description (.. info (getDescription) (getContent)) + :upload-date (.getTextualUploadDate info) + :upload-author (.getUploaderName info) + :upload-url (.getUploaderUrl info) + :upload-avatar (.getUploaderAvatarUrl info) + :upload-verified? (.isUploaderVerified info) + :service-id (.getServiceId info) + :thumbnail-url (.getThumbnailUrl info) + :duration (.getDuration info) + :tags (.getTags info) + :category (.getCategory info) + :view-count (.getViewCount info) + :like-count (.getLikeCount info) + :dislike-count (.getDislikeCount info) + :subscriber-count (.getUploaderSubscriberCount info) + :audio-streams (j/from-java (.getAudioStreams info)) + :video-streams (j/from-java (.getVideoStreams info)) + :hls-url (.getHlsUrl info) + :dash-mpd-url (.getDashMpdUrl info) + :related-streams (map #(get-stream-result %) (.getRelatedStreams info))}))) diff --git a/src/backend/tau/core.clj b/src/backend/tau/core.clj new file mode 100644 index 0000000..8fb8b07 --- /dev/null +++ b/src/backend/tau/core.clj @@ -0,0 +1,11 @@ +(ns tau.core + (:require + [tau.services.http :as http])) + +(defn -main + [& _] + (http/start-server! 3000)) + +(defn reset + [] + (http/stop-server!)) diff --git a/src/backend/tau/downloader_impl.clj b/src/backend/tau/downloader_impl.clj new file mode 100644 index 0000000..8a62821 --- /dev/null +++ b/src/backend/tau/downloader_impl.clj @@ -0,0 +1,64 @@ +(ns tau.downloader-impl + (:import + [org.schabi.newpipe.extractor.downloader Response Request] + [okhttp3 Request$Builder OkHttpClient$Builder RequestBody]) + (:gen-class + :extends org.schabi.newpipe.extractor.downloader.Downloader + :constructors {[okhttp3.OkHttpClient$Builder] []} + :name tau.DownloaderImpl + :init downloader-impl + :state state + :methods [#^{:static true} [init [] tau.DownloaderImpl] + #^{:static true} [getInstance [] tau.DownloaderImpl]] + :prefix "-" + :main false)) + +(def user-agent "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0") +(def instance (atom nil)) + +(defn -downloader-impl [builder] + [[] (atom {:client + (.. builder + (readTimeout 30 (java.util.concurrent.TimeUnit/SECONDS)) + (build))})]) + +(defn -init + ([] + (-init (OkHttpClient$Builder.))) + ([builder] + (reset! instance (tau.DownloaderImpl. builder)))) + +(defn -getInstance [] + (if @instance @instance (-init))) + +(defn -execute [this request] + (let [http-method (.httpMethod request) + url (.url request) + headers (.headers request) + data-to-send (.dataToSend request) + request-body (when data-to-send (RequestBody/create nil data-to-send)) + request-builder (.. (Request$Builder.) + (method http-method request-body) + (url url) + (addHeader "User-Agent" user-agent))] + (doseq [pair (.entrySet headers)] + (let [header-name (.getKey pair) + header-value-list (.getValue pair)] + (if (> (.size header-value-list) 1) + (do + (.removeHeader request-builder header-name) + (doseq [header-value header-value-list] + (.addHeader request-builder header-name header-value))) + (if (= (.size header-value-list) 1) + (.header request-builder header-name (.get header-value-list 0)))))) + (let [response (.. (@(.state this) :client) (newCall (.build request-builder)) (execute)) + body (.body response) + response-body-to-return (when body (.string body)) + latest-url (.. response (request) (url) (toString))] + (when (= (.code response) 429) + (.close response)) + (Response. (.code response) + (.message response) + (.. response (headers) (toMultimap)) + response-body-to-return + latest-url)))) diff --git a/src/backend/tau/services/http.clj b/src/backend/tau/services/http.clj new file mode 100644 index 0000000..6101b52 --- /dev/null +++ b/src/backend/tau/services/http.clj @@ -0,0 +1,73 @@ +(ns tau.services.http + (:require + [org.httpkit.server :refer [run-server]] + [ring.middleware.reload :refer [wrap-reload]] + [ring.middleware.params :refer [wrap-params]] + [ring.middleware.json :refer [wrap-json-response]] + [ring.util.response :refer [response]] + [compojure.route :as route] + [compojure.core :refer :all] + [compojure.coercions :refer [as-int]] + [clojure.string :as str] + [tau.api.stream :as stream] + [tau.api.search :as search] + [tau.api.channel :as channel] + [tau.api.playlist :as playlist] + [tau.api.comment :as comment] + [tau.api.kiosk :as kiosk] + [tau.api.service :as service]) + (:import + tau.DownloaderImpl + org.schabi.newpipe.extractor.NewPipe + org.schabi.newpipe.extractor.localization.Localization)) + +(defonce server (atom nil)) + +(defn stop-server! + [] + (when @server + (@server :timeout 100) + (reset! server nil))) + +(defroutes app-routes + (context "/api" [] + (GET "/stream" [url] + (response (stream/get-stream-info url))) + (GET "/search" [serviceId :<< as-int q sortFilter contentFilters nextPage] + (let [content-filters (when contentFilters (str/split contentFilters #","))] + (response (if nextPage + (search/get-search-info serviceId q content-filters sortFilter nextPage) + (search/get-search-info serviceId q content-filters sortFilter))))) + (GET "/channel" [url nextPage] + (if nextPage + (response (channel/get-channel-info url nextPage)) + (response (channel/get-channel-info url)))) + (GET "/playlist" [url nextPage] + (if nextPage + (response (playlist/get-playlist-info url nextPage)) + (response (playlist/get-playlist-info url)))) + (GET "/comments" [url nextPage] + (if nextPage + (response (comment/get-comments-info url nextPage)) + (response (comment/get-comments-info url)))) + (GET "/services" [] + (response (service/get-service-list-info))) + (context "/kiosks" [] + (GET "/" [serviceId :<< as-int] + (response (kiosk/get-kiosk-list-info serviceId))) + (GET "/:kioskId" [kioskId serviceId :<< as-int nextPage] + (if nextPage + (response (kiosk/get-kiosk-info kioskId serviceId nextPage)) + (response (kiosk/get-kiosk-info kioskId serviceId))))))) + +(defn make-handler + [] + (-> #'app-routes + wrap-params + (wrap-json-response {:pretty true}) + wrap-reload)) + +(defn start-server! + [port] + (NewPipe/init (DownloaderImpl/init) (Localization. "en" "GB")) + (reset! server (run-server (make-handler) {:port port}))) diff --git a/src/backend/tau/utils.clj b/src/backend/tau/utils.clj new file mode 100644 index 0000000..41c0086 --- /dev/null +++ b/src/backend/tau/utils.clj @@ -0,0 +1 @@ +(ns tau.utils) -- cgit v1.2.3