added omlp support, query optimizations

pull/24/head
Akhil Gupta 4 years ago
parent 0ef179c370
commit 68dd9179eb

@ -1,55 +1,112 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Add Podcast - PodGrab</title>
{{template "commoncss"}}
<style>
[v-cloak] { display: none }
[v-cloak] {
display: none;
}
</style>
</head>
<body>
<div class="container">
{{template "navbar" .}}
<br>
<br />
<div id="app" v-cloak>
<div class="row">
<h4>Add using the direct link to rss feed</h4>
<form action="/" method="post" @submit="addPodcastManual">
<div class="nine columns">
<input type="url" v-model="url" name="url" id="url" placeholder="Enter Podcast RSS feed to add" class="u-full-width">
<input
type="url"
v-model="url"
name="url"
id="url"
placeholder="Enter Podcast RSS feed to add"
class="u-full-width"
/>
</div>
<div class="three columns">
<input type="submit" value="Add Podcast" class="u-full-width button-primary">
<input
type="submit"
value="Add Podcast"
class="u-full-width button-primary"
/>
</div>
</form>
</div>
<hr>
<hr />
<div class="row">
<div>
<h4>Import OPML file</h4>
<i
><small
>Most of the major podcast manager apps (eg. Podcast Addict) have the option to export add your podcast subscriptions in the opml format. You can migrate all your podcast in one go by exportin that OPML file and importing it here.</small
></i
>
</div><br>
<form enctype="multipart/form-data" action="/" @submit="uploadOpml" ref="uploadForm">
<div class="nine columns">
<input
type="file"
:name="uploadFieldName"
@change="selectFile"
ref="file"
accept="text/xml,.opml"
/>
</div>
<div class="three columns">
<input
type="submit"
value="Upload"
class="u-full-width button-primary"
/>
</div> </form>
</div>
<hr />
<div class="row" id="searchContainer">
<h4>Search for your favorite podcast</h4>
<i><small>Experimental: Uses iTunes API to show search results.</small></i>
<br>
<i
><small
>Experimental: Uses iTunes API to show search results.</small
></i
>
<br />
<form action="/search" method="post" @submit="search">
<div class="nine columns">
<input type="search" name="search" id="search" placeholder="Search for your podcast" v-model="query" class="u-full-width">
<input
type="search"
name="search"
id="search"
placeholder="Search for your podcast"
v-model="query"
class="u-full-width"
/>
</div>
<div class="three columns">
<input type="submit" value="Search" class="u-full-width button-primary">
<input
type="submit"
value="Search"
class="u-full-width button-primary"
/>
</div>
</form>
<br>
<br />
<progress v-if="searching" class="u-full-width"></progress>
<div class="results">
<div v-for="item in results" :key="item.url">
<div class="row">
<div class="columns two">
<img class="u-full-width" :src="item.image" :alt="item.title">
<img
class="u-full-width"
:src="item.image"
:alt="item.title"
/>
</div>
<div class="columns nine">
<h5>${item.title}</h5>
@ -57,12 +114,23 @@
<p>${ item.description }</p>
</div>
<div class="columns one">
<button v-if="!item.already_saved" v-on:click="addPodcast(item)" class="button button-primary">+ Add</button >
<button v-if="item.already_saved" class="button" disabled="disabled">Already Added</button >
<button
v-if="!item.already_saved"
v-on:click="addPodcast(item)"
class="button button-primary"
>
+ Add
</button>
<button
v-if="item.already_saved"
class="button"
disabled="disabled"
>
Already Added
</button>
</div>
</div>
<hr>
<hr />
</div>
</div>
</div>
@ -72,78 +140,116 @@
<script>
var app = new Vue({
delimiters: ['${', '}'],
el: '#app',
delimiters: ["${", "}"],
el: "#app",
data: {
results: [],
query:'',
query: "",
searching: false,
url:''
url: "",
selectedFiles: undefined,
},
methods: {
search:function(e){
selectFile: function () {
this.selectedFiles = this.$refs.file.files;
},
uploadOpml: function (e) {
e.preventDefault();
if(!this.query){
var currentFile = this.selectedFiles.item(0);
if (!currentFile) {
return;
}
var self=this;
self.searching = true;
axios.get("/search?q="+this.query)
let formData = new FormData();
formData.append("file", currentFile);
axios
.post("/opml", formData, {
headers: {
"Content-Type": "multipart/form-data",
},
})
.then(function (response) {
self.results= response.data
Vue.toasted.show("File uploaded successfully.", {
theme: "bubble",
position: "top-right",
duration: 5000,
});
item.already_saved = true;
})
.catch(function (error) {
}).
then(function(){
self.searching=false;
if (error.response) {
Vue.toasted.show(error.response.data?.message, {
theme: "bubble",
position: "top-right",
duration: 5000,
});
}
})
.then(function () {
self.searching = false;
self.$refs.uploadForm.reset()
});
},
search: function (e) {
e.preventDefault();
if (!this.query) {
return;
}
var self = this;
self.searching = true;
axios
.get("/search?q=" + this.query)
.then(function (response) {
self.results = response.data;
})
.catch(function (error) {})
.then(function () {
self.searching = false;
});
},
addPodcastManual: function (e) {
e.preventDefault();
if (!this.url) {
return;
}
this.addPodcast({url:this.url})
}
,
this.addPodcast({ url: this.url });
},
addPodcast: function (item) {
// console.log(item);
var self = this;
self.searching = true;
axios.post("/podcasts",{
url:item.url
axios
.post("/podcasts", {
url: item.url,
})
.then(function (response) {
Vue.toasted.show('Podcast added successfully.' ,{
Vue.toasted.show("Podcast added successfully.", {
theme: "bubble",
position: "top-right",
duration : 5000
})
item.already_saved= true
duration: 5000,
});
item.already_saved = true;
})
.catch(function (error) {
if (error.response) {
Vue.toasted.show(error.response.data?.message, {
theme: "bubble",
position: "top-right",
duration : 5000
})
duration: 5000,
});
}
}).
then(function(){
self.searching=false;
self.url=""
})
.then(function () {
self.searching = false;
self.url = "";
});
return false;
}
}
})
},
},
});
</script>
</body>
</html>

@ -88,10 +88,10 @@ hr{
<div class="row"><div class="columns twelve" style="text-align: center;">
{{if .previousPage }}
<a href="?page={{.previousPage}}" class="button button-primary">Newer</a>
<a href="?page={{.previousPage}}&downloadedOnly={{.downloadedOnly}}" class="button button-primary">Newer</a>
{{end}}
{{if .nextPage }}
<a href="?page={{.nextPage}}" class="button button-primary">Older</a>
<a href="?page={{.nextPage}}&downloadedOnly={{.downloadedOnly}}" class="button button-primary">Older</a>
{{end}}
</div></div>

@ -1,7 +1,9 @@
package controllers
import (
"bytes"
"fmt"
"io"
"math"
"net/http"
"os"
@ -50,6 +52,7 @@ func PodcastPage(c *gin.Context) {
"totalPages": 0,
"nextPage": 0,
"previousPage": 0,
"downloadedOnly": false,
})
} else {
c.JSON(http.StatusBadRequest, err)
@ -111,7 +114,7 @@ func AllEpisodesPage(c *gin.Context) {
}
var podcastItems []db.PodcastItem
var totalCount int64
if err := db.GetPaginatedPodcastItems(page, count, &podcastItems, &totalCount); err == nil {
if err := db.GetPaginatedPodcastItems(page, count, pagination.DownloadedOnly, &podcastItems, &totalCount); err == nil {
setting := c.MustGet("setting").(*db.Setting)
totalPages := math.Ceil(float64(totalCount) / float64(count))
nextPage, previousPage := 0, 0
@ -131,12 +134,13 @@ func AllEpisodesPage(c *gin.Context) {
"totalPages": totalPages,
"nextPage": nextPage,
"previousPage": previousPage,
"downloadedOnly": pagination.DownloadedOnly,
})
} else {
c.JSON(http.StatusBadRequest, err)
}
} else {
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request"})
c.JSON(http.StatusBadRequest, gin.H{"message": "Invalid request"})
}
}
@ -162,6 +166,27 @@ func Search(c *gin.Context) {
}
}
func UploadOpml(c *gin.Context) {
file, _, err := c.Request.FormFile("file")
defer file.Close()
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"message": "Invalid request"})
return
}
buf := bytes.NewBuffer(nil)
if _, err := io.Copy(buf, file); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"message": "Invalid request"})
return
}
content := string(buf.Bytes())
err = service.AddOpml(content)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"message": err.Error()})
} else {
c.JSON(200, gin.H{"success": "File uploaded"})
}
}
func AddNewPodcast(c *gin.Context) {
var addPodcastData AddPodcastData

@ -24,6 +24,7 @@ type SearchByIdQuery struct {
type Pagination struct {
Page int `uri:"page" query:"page" json:"page" form:"page"`
Count int `uri:"count" query:"count" json:"count" form:"count"`
DownloadedOnly bool `uri:"downloadedOnly" query:"downloadedOnly" json:"downloadedOnly" form:"downloadedOnly"`
}
type AddPodcastData struct {

@ -26,8 +26,13 @@ func GetAllPodcastItems(podcasts *[]PodcastItem) error {
result := DB.Preload("Podcast").Order("pub_date desc").Find(&podcasts)
return result.Error
}
func GetPaginatedPodcastItems(page int, count int, podcasts *[]PodcastItem, total *int64) error {
result := DB.Debug().Preload("Podcast").Limit(count).Offset((page - 1) * count).Order("pub_date desc").Find(&podcasts)
func GetPaginatedPodcastItems(page int, count int, downloadedOnly bool, podcasts *[]PodcastItem, total *int64) error {
query := DB.Debug().Preload("Podcast")
if downloadedOnly {
query = query.Where("download_status=?", Downloaded)
}
result := query.Limit(count).Offset((page - 1) * count).Order("pub_date desc").Find(&podcasts)
DB.Model(&PodcastItem{}).Count(total)
@ -75,6 +80,11 @@ func GetAllPodcastItemsAlreadyDownloaded() (*[]PodcastItem, error) {
return &podcastItems, result.Error
}
func GetPodcastItemsByPodcastIdAndGUIDs(podcastId string, guids []string) (*[]PodcastItem, error) {
var podcastItems []PodcastItem
result := DB.Preload(clause.Associations).Where(&PodcastItem{PodcastID: podcastId}).Where("guid IN ?", guids).Find(&podcastItems)
return &podcastItems, result.Error
}
func GetPodcastItemByPodcastIdAndGUID(podcastId string, guid string, podcastItem *PodcastItem) error {
result := DB.Preload(clause.Associations).Where(&PodcastItem{PodcastID: podcastId, GUID: guid}).First(&podcastItem)

@ -91,6 +91,7 @@ func main() {
r.GET("/settings", controllers.SettingsPage)
r.POST("/settings", controllers.UpdateSetting)
r.GET("/backups", controllers.BackupsPage)
r.POST("/opml", controllers.UploadOpml)
go assetEnv()
go intiCron()

@ -0,0 +1,30 @@
package model
import "encoding/xml"
type OpmlModel struct {
XMLName xml.Name `xml:"opml"`
Text string `xml:",chardata"`
Version string `xml:"version,attr"`
Head struct {
Text string `xml:",chardata"`
Title string `xml:"title"`
} `xml:"head"`
Body struct {
Text string `xml:",chardata"`
Outline []struct {
Text string `xml:",chardata"`
Title string `xml:"title,attr"`
AttrText string `xml:"text,attr"`
Type string `xml:"type,attr"`
XmlUrl string `xml:"xmlUrl,attr"`
Outline []struct {
Text string `xml:",chardata"`
AttrText string `xml:"text,attr"`
Title string `xml:"title,attr"`
Type string `xml:"type,attr"`
XmlUrl string `xml:"xmlUrl,attr"`
} `xml:"outline"`
} `xml:"outline"`
} `xml:"body"`
}

@ -19,10 +19,13 @@ import (
)
func Download(link string, episodeTitle string, podcastName string) (string, error) {
if link == "" {
return "", errors.New("Download path empty")
}
client := httpClient()
resp, err := client.Get(link)
if err != nil {
Logger.Errorw("Error getting response", err)
Logger.Errorw("Error getting response: "+link, err)
return "", err
}
@ -31,7 +34,7 @@ func Download(link string, episodeTitle string, podcastName string) (string, err
finalPath := path.Join(folder, fileName)
file, err := os.Create(finalPath)
if err != nil {
Logger.Errorw("Error creating file", err)
Logger.Errorw("Error creating file"+link, err)
return "", err
}
defer resp.Body.Close()
@ -39,7 +42,7 @@ func Download(link string, episodeTitle string, podcastName string) (string, err
//fmt.Println(size)
defer file.Close()
if erra != nil {
Logger.Errorw("Error saving file", err)
Logger.Errorw("Error saving file"+link, err)
return "", erra
}
changeOwnership(finalPath)

@ -8,6 +8,7 @@ import (
"net/http"
"os"
"strconv"
"sync"
"time"
"github.com/akhilrex/podgrab/db"
@ -25,6 +26,12 @@ func init() {
defer zapper.Sync()
}
func ParseOpml(content string) (model.OpmlModel, error) {
var response model.OpmlModel
err := xml.Unmarshal([]byte(content), &response)
return response, err
}
//FetchURL is
func FetchURL(url string) (model.PodcastData, error) {
body, err := makeQuery(url)
@ -40,9 +47,45 @@ func GetAllPodcasts() *[]db.Podcast {
db.GetAllPodcasts(&podcasts)
return &podcasts
}
func AddOpml(content string) error {
model, err := ParseOpml(content)
if err != nil {
return errors.New("Invalid file format")
}
var wg sync.WaitGroup
setting := db.GetOrCreateSetting()
for _, outline := range model.Body.Outline {
if outline.XmlUrl != "" {
wg.Add(1)
go func(url string) {
defer wg.Done()
AddPodcast(url)
}(outline.XmlUrl)
}
for _, innerOutline := range outline.Outline {
if innerOutline.XmlUrl != "" {
wg.Add(1)
go func(url string) {
defer wg.Done()
AddPodcast(url)
}(innerOutline.XmlUrl)
}
}
}
wg.Wait()
if setting.DownloadOnAdd {
go RefreshEpisodes()
}
return nil
}
func AddPodcast(url string) (db.Podcast, error) {
var podcast db.Podcast
err := db.GetPodcastByURL(url, &podcast)
fmt.Println(url)
if errors.Is(err, gorm.ErrRecordNotFound) {
data, err := FetchURL(url)
if err != nil {
@ -77,11 +120,24 @@ func AddPodcastItems(podcast *db.Podcast) error {
// if len(data.Channel.Item) < limit {
// limit = len(data.Channel.Item)
// }
var allGuids []string
for i := 0; i < len(data.Channel.Item); i++ {
obj := data.Channel.Item[i]
allGuids = append(allGuids, obj.Guid.Text)
}
existingItems, err := db.GetPodcastItemsByPodcastIdAndGUIDs(podcast.ID, allGuids)
keyMap := make(map[string]int)
for _, item := range *existingItems {
keyMap[item.GUID] = 1
}
for i := 0; i < len(data.Channel.Item); i++ {
obj := data.Channel.Item[i]
var podcastItem db.PodcastItem
err := db.GetPodcastItemByPodcastIdAndGUID(podcast.ID, obj.Guid.Text, &podcastItem)
if errors.Is(err, gorm.ErrRecordNotFound) {
_, keyExists := keyMap[obj.Guid.Text]
if !keyExists {
duration, _ := strconv.Atoi(obj.Duration)
pubDate, _ := time.Parse(time.RFC1123Z, obj.PubDate)
if (pubDate == time.Time{}) {
@ -143,11 +199,20 @@ func DownloadMissingEpisodes() error {
if err != nil {
return err
}
for _, item := range *data {
var wg sync.WaitGroup
for index, item := range *data {
wg.Add(1)
go func(item db.PodcastItem) {
defer wg.Done()
url, _ := Download(item.FileURL, item.Title, item.Podcast.Title)
SetPodcastItemAsDownloaded(item.ID, url)
}(item)
if index%5 == 0 {
wg.Wait()
}
}
wg.Wait()
return nil
}
func CheckMissingFiles() error {

Loading…
Cancel
Save