From aa434d591e8ad80c2f750221d2b44189a8e0ccda Mon Sep 17 00:00:00 2001 From: softworkz Date: Tue, 26 Jul 2016 05:29:05 +0200 Subject: [PATCH 1/4] Add StringUsageReporter StringUsageReporter reports usages of localization strings in the web application and can also list all unused strings --- .../Resources/SampleTransformed.htm | 1348 +++++++++++++++++ .../Resources/StringCheck.xslt | 145 ++ .../Resources/StringCheckSample.xml | 239 +++ .../ConsistencyTests/StringUsageReporter.cs | 260 ++++ .../TextIndexing/IndexBuilder.cs | 54 + .../TextIndexing/WordIndex.cs | 39 + .../TextIndexing/WordOccurrence.cs | 24 + .../TextIndexing/WordOccurrences.cs | 17 + MediaBrowser.Tests/MediaBrowser.Tests.csproj | 15 + 9 files changed, 2141 insertions(+) create mode 100644 MediaBrowser.Tests/ConsistencyTests/Resources/SampleTransformed.htm create mode 100644 MediaBrowser.Tests/ConsistencyTests/Resources/StringCheck.xslt create mode 100644 MediaBrowser.Tests/ConsistencyTests/Resources/StringCheckSample.xml create mode 100644 MediaBrowser.Tests/ConsistencyTests/StringUsageReporter.cs create mode 100644 MediaBrowser.Tests/ConsistencyTests/TextIndexing/IndexBuilder.cs create mode 100644 MediaBrowser.Tests/ConsistencyTests/TextIndexing/WordIndex.cs create mode 100644 MediaBrowser.Tests/ConsistencyTests/TextIndexing/WordOccurrence.cs create mode 100644 MediaBrowser.Tests/ConsistencyTests/TextIndexing/WordOccurrences.cs diff --git a/MediaBrowser.Tests/ConsistencyTests/Resources/SampleTransformed.htm b/MediaBrowser.Tests/ConsistencyTests/Resources/SampleTransformed.htm new file mode 100644 index 0000000000..9bff396d13 --- /dev/null +++ b/MediaBrowser.Tests/ConsistencyTests/Resources/SampleTransformed.htm @@ -0,0 +1,1348 @@ + + + + + String Usage Report + + + +

String Usage Report

+
+

Strings

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ LabelExit: "
+
Exit"
+
+
+
+
+ LabelVisitCommunity: "
+
Visit Community"
+
+
+
+
+ LabelGithub: "
+
Github"
+
+
+
+
+ LabelSwagger: "
+
Swagger"
+
+
+
+
+ LabelStandard: "
+
Standard"
+
+
+
+
+ LabelApiDocumentation: "
+
Api Documentation"
+
+
+
+
+ LabelDeveloperResources: "
+
Developer Resources"
+
+
+
+
+ LabelBrowseLibrary: "
+
Browse Library"
+
+
+
+
+ LabelConfigureServer: "
+
Configure Emby"
+
+
+
+
+ LabelOpenLibraryViewer: "
+
Open Library Viewer"
+
+
+
+
+ LabelRestartServer: "
+
Restart Server"
+
+
+
+
+ LabelShowLogWindow: "
+
Show Log Window"
+
+
+
+
+ LabelPrevious: "
+
Previous"
+
+
+ \wizardagreement.html:21 +
+ \wizardcomponents.html:54 +
+ \wizardfinish.html:40 +
+ \wizardlibrary.html:19 +
+ \wizardlivetvguide.html:30 +
+ \wizardlivetvtuner.html:31 +
+ \wizardservice.html:17 +
+ \wizardsettings.html:32 +
+ \wizarduser.html:27 +
+
+
+ LabelFinish: "
+
Finish"
+
+
+ \wizardfinish.html:41 +
+
+
+ LabelNext: "
+
Next"
+
+
+ \wizardagreement.html:22 +
+ \wizardcomponents.html:55 +
+ \wizardlibrary.html:20 +
+ \wizardlivetvguide.html:31 +
+ \wizardlivetvtuner.html:32 +
+ \wizardservice.html:18 +
+ \wizardsettings.html:33 +
+ \wizardstart.html:25 +
+ \wizarduser.html:28 +
+
+
+ LabelYoureDone: "
+
You're Done!"
+
+
+ \wizardfinish.html:7 +
+
+
+ WelcomeToProject: "
+
Welcome to Emby!"
+
+
+ \wizardstart.html:10 +
+
+
+ ThisWizardWillGuideYou: "
+
This wizard will help guide you through the setup process. To begin, please select your preferred language."
+
+
+ \wizardstart.html:16 +
+
+
+ TellUsAboutYourself: "
+
Tell us about yourself"
+
+
+ \wizarduser.html:8 +
+
+
+ ButtonQuickStartGuide: "
+
Quick start guide"
+
+
+ \wizardstart.html:12 +
+
+
+ LabelYourFirstName: "
+
Your first name:"
+
+
+ \wizarduser.html:14 +
+
+
+ MoreUsersCanBeAddedLater: "
+
More users can be added later within the Dashboard."
+
+
+ \wizarduser.html:15 +
+
+
+ UserProfilesIntro: "
+
Emby includes built-in support for user profiles, enabling each user to have their own display settings, playstate and parental controls."
+
+
+ \wizarduser.html:11 +
+
+
+ LabelWindowsService: "
+
Windows Service"
+
+
+ \wizardservice.html:7 +
+
+
+ AWindowsServiceHasBeenInstalled: "
+
A Windows Service has been installed."
+
+
+ \wizardservice.html:10 +
+
+
+ WindowsServiceIntro1: "
+
Emby Server normally runs as a desktop application with a tray icon, but if you prefer to run it as a background service, it can be started from the windows services control panel instead."
+
+
+ \wizardservice.html:12 +
+
+
+ WindowsServiceIntro2: "
+
If using the windows service, please note that it cannot be run at the same time as the tray icon, so you'll need to exit the tray in order to run the service. The service will also need to be configured with administrative privileges via the control panel. When running as a service, you will need to ensure that the service account has access to your media folders."
+
+
+ \wizardservice.html:14 +
+
+
+ WizardCompleted: "
+
That's all we need for now. Emby has begun collecting information about your media library. Check out some of our apps, and then click <b>Finish</b> to view the <b>Server Dashboard</b>."
+
+
+ \wizardfinish.html:10 +
+
+
+ LabelConfigureSettings: "
+
Configure settings"
+
+
+ \wizardsettings.html:8 +
+
+
+ LabelEnableVideoImageExtraction: "
+
Enable video image extraction"
+
+
+
+
+ VideoImageExtractionHelp: "
+
For videos that don't already have images, and that we're unable to find internet images for. This will add some additional time to the initial library scan but will result in a more pleasing presentation."
+
+
+
+
+ LabelEnableChapterImageExtractionForMovies: "
+
Extract chapter image extraction for Movies"
+
+
+
+
+ LabelChapterImageExtractionForMoviesHelp: "
+
Extracting chapter images will allow clients to display graphical scene selection menus. The process can be slow, cpu-intensive and may require several gigabytes of space. It runs as a nightly scheduled task, although this is configurable in the scheduled tasks area. It is not recommended to run this task during peak usage hours."
+
+
+
+
+ LabelEnableAutomaticPortMapping: "
+
Enable automatic port mapping"
+
+
+
+
+ LabelEnableAutomaticPortMappingHelp: "
+
UPnP allows automated router configuration for easy remote access. This may not work with some router models."
+
+
+
+
+ HeaderTermsOfService: "
+
Emby Terms of Service"
+
+
+ \wizardagreement.html:9 +
+
+
+ MessagePleaseAcceptTermsOfService: "
+
Please accept the terms of service and privacy policy before continuing."
+
+
+ \wizardagreement.html:12 +
+
+
+ OptionIAcceptTermsOfService: "
+
I accept the terms of service"
+
+
+ \wizardagreement.html:17 +
+
+
+ ButtonPrivacyPolicy: "
+
Privacy policy"
+
+
+ \wizardagreement.html:14 +
+
+
+ ButtonTermsOfService: "
+
Terms of Service"
+
+
+ \wizardagreement.html:15 +
+
+
+ HeaderDeveloperOptions: "
+
Developer Options"
+
+
+ \dashboardgeneral.html:108 +
+
+
+ OptionEnableWebClientResponseCache: "
+
Enable web response caching"
+
+
+ \dashboardgeneral.html:112 +
+
+
+ OptionDisableForDevelopmentHelp: "
+
Configure these as needed for web development purposes."
+
+
+ \dashboardgeneral.html:119 +
+
+
+ OptionEnableWebClientResourceMinification: "
+
Enable web resource minification"
+
+
+ \dashboardgeneral.html:116 +
+
+
+ LabelDashboardSourcePath: "
+
Web client source path:"
+
+
+ \dashboardgeneral.html:124 +
+
+
+ LabelDashboardSourcePathHelp: "
+
If running the server from source, specify the path to the dashboard-ui folder. All web client files will be served from this location."
+
+
+ \dashboardgeneral.html:126 +
+
+
+ ButtonConvertMedia: "
+
Convert media"
+
+
+ \syncactivity.html:22 +
+
+
+ ButtonOrganize: "
+
Organize"
+
+
+ \autoorganizelog.html:8 +
+ \scripts\autoorganizelog.js:293 +
+ \scripts\autoorganizelog.js:294 +
+ \scripts\autoorganizelog.js:296 +
+
+
+ LinkedToEmbyConnect: "
+
Linked to Emby Connect"
+
+
+
+
+ HeaderSupporterBenefits: "
+
Emby Premiere Benefits"
+
+
+
+
+ HeaderAddUser: "
+
Add User"
+
+
+
+
+ LabelAddConnectSupporterHelp: "
+
To add a user who isn't listed, you'll need to first link their account to Emby Connect from their user profile page."
+
+
+
+
+ LabelPinCode: "
+
Pin code:"
+
+
+
+
+ OptionHideWatchedContentFromLatestMedia: "
+
Hide watched content from latest media"
+
+
+ \mypreferenceshome.html:114 +
+
+
+ HeaderSync: "
+
Sync"
+
+
+ \mysyncsettings.html:7 +
+ \scripts\registrationservices.js:175 +
+ \useredit.html:82 +
+
+
+ ButtonOk: "
+
Ok"
+
+
+ \components\directorybrowser\directorybrowser.js:147 +
+ \components\fileorganizer\fileorganizer.template.html:45 +
+ \components\medialibrarycreator\medialibrarycreator.template.html:30 +
+ \components\metadataeditor\personeditor.template.html:33 +
+ \dlnaprofile.html:372 +
+ \dlnaprofile.html:453 +
+ \dlnaprofile.html:504 +
+ \dlnaprofile.html:542 +
+ \dlnaprofile.html:590 +
+ \dlnaprofile.html:630 +
+ \dlnaprofile.html:661 +
+ \dlnaprofile.html:706 +
+ \nowplaying.html:113 +
+ \scripts\ratingdialog.js:42 +
+
+
+ ButtonCancel: "
+
Cancel"
+
+
+ \components\tvproviders\schedulesdirect.template.html:68 +
+ \components\tvproviders\xmltv.template.html:48 +
+ \connectlogin.html:74 +
+ \connectlogin.html:108 +
+ \dlnaprofile.html:325 +
+ \dlnaprofile.html:375 +
+ \dlnaprofile.html:456 +
+ \dlnaprofile.html:507 +
+ \dlnaprofile.html:545 +
+ \dlnaprofile.html:593 +
+ \dlnaprofile.html:633 +
+ \dlnaprofile.html:664 +
+ \dlnaprofile.html:709 +
+ \forgotpassword.html:23 +
+ \forgotpasswordpin.html:22 +
+ \livetvseriestimer.html:62 +
+ \livetvtunerprovider-hdhomerun.html:35 +
+ \livetvtunerprovider-m3u.html:19 +
+ \livetvtunerprovider-satip.html:65 +
+ \login.html:27 +
+ \notificationsetting.html:64 +
+ \scheduledtask.html:85 +
+ \scripts\librarylist.js:349 +
+ \scripts\mediacontroller.js:167 +
+ \scripts\mediacontroller.js:436 +
+ \scripts\ratingdialog.js:43 +
+ \scripts\site.js:1025 +
+ \scripts\userprofilespage.js:198 +
+ \syncsettings.html:43 +
+ \useredit.html:111 +
+ \userlibraryaccess.html:57 +
+ \usernew.html:45 +
+ \userparentalcontrol.html:101 +
+
+
+ ButtonExit: "
+
Exit"
+
+
+
+
+ ButtonNew: "
+
New"
+
+
+ \components\fileorganizer\fileorganizer.template.html:18 +
+ \dlnaprofile.html:107 +
+ \dlnaprofile.html:278 +
+ \dlnaprofile.html:290 +
+ \dlnaprofile.html:296 +
+ \dlnaprofile.html:302 +
+ \dlnaprofile.html:308 +
+ \dlnaprofile.html:314 +
+ \dlnaprofiles.html:14 +
+ \serversecurity.html:8 +
+
+
+ HeaderTaskTriggers: "
+
Task Triggers"
+
+
+ \scheduledtask.html:11 +
+
+
+ HeaderTV: "
+
TV"
+
+
+ \librarysettings.html:113 +
+
+
+ HeaderAudio: "
+
Audio"
+
+
+ \librarysettings.html:39 +
+
+
+ HeaderVideo: "
+
Video"
+
+
+ \librarysettings.html:50 +
+
+
+ HeaderPaths: "
+
Paths"
+
+
+ \dashboard.html:92 +
+
+
+ CategorySync: "
+
Sync"
+
+
+
+
+ TabPlaylist: "
+
Playlist"
+
+
+ \nowplaying.html:20 +
+
+
+ HeaderEasyPinCode: "
+
Easy Pin Code"
+
+
+ \myprofile.html:69 +
+ \userpassword.html:42 +
+
+
+ HeaderGrownupsOnly: "
+
Grown-ups Only!"
+
+
+
+
+ DividerOr: "
+
-- or --"
+
+
+
+
+ HeaderInstalledServices: "
+
Installed Services"
+
+
+ \appservices.html:6 +
+
+
+ HeaderAvailableServices: "
+
Available Services"
+
+
+ \appservices.html:11 +
+
+
+ + \ No newline at end of file diff --git a/MediaBrowser.Tests/ConsistencyTests/Resources/StringCheck.xslt b/MediaBrowser.Tests/ConsistencyTests/Resources/StringCheck.xslt new file mode 100644 index 0000000000..39586022b3 --- /dev/null +++ b/MediaBrowser.Tests/ConsistencyTests/Resources/StringCheck.xslt @@ -0,0 +1,145 @@ + + + + + + + + + + + + +]> + + + + + + + + <xsl:value-of select="StringUsages/@ReportTitle"/> + + + + +

+ +

+
+

Strings

+
+ + + + + + + + + + + + + + +
+
: "
+
"
+
:
+
+
+ + + + \ No newline at end of file diff --git a/MediaBrowser.Tests/ConsistencyTests/Resources/StringCheckSample.xml b/MediaBrowser.Tests/ConsistencyTests/Resources/StringCheckSample.xml new file mode 100644 index 0000000000..118ed55aef --- /dev/null +++ b/MediaBrowser.Tests/ConsistencyTests/Resources/StringCheckSample.xml @@ -0,0 +1,239 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/MediaBrowser.Tests/ConsistencyTests/StringUsageReporter.cs b/MediaBrowser.Tests/ConsistencyTests/StringUsageReporter.cs new file mode 100644 index 0000000000..d036a6c6d7 --- /dev/null +++ b/MediaBrowser.Tests/ConsistencyTests/StringUsageReporter.cs @@ -0,0 +1,260 @@ +using MediaBrowser.Tests.ConsistencyTests.TextIndexing; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Xml; + +namespace MediaBrowser.Tests.ConsistencyTests +{ + /// + /// This class contains tests for reporting the usage of localization string tokens + /// in the dashboard-ui or similar. + /// + /// + /// Run one of the two tests using Visual Studio's "Test Explorer": + /// + /// + /// + /// + /// + /// + /// + /// On successful run, the bottom section of the test explorer will contain a link "Output". + /// This link will open the test results, displaying the trace and two attachment links. + /// One link will open the output folder, the other link will open the output xml file. + /// + /// + /// The output xml file contains a stylesheet link to render the results as html. + /// How that works depends on the default application configured for XML files: + /// + /// + /// Visual Studio + /// Will open in XML source view. To view the html result, click menu + /// 'XML' => 'Start XSLT without debugging' + /// Internet Explorer + /// XSL transform will be applied automatically. + /// Firefox + /// XSL transform will be applied automatically. + /// Chrome + /// Does not work. Chrome is unable/unwilling to apply xslt transforms from local files. + /// + /// + [TestClass] + public class StringUsageReporter + { + /// + /// Root path of the web application + /// + /// + /// Can be an absolute path or a path relative to the binaries folder (bin\Debug). + /// + public const string WebFolder = @"..\..\..\MediaBrowser.WebDashboard\dashboard-ui"; + + /// + /// Path to the strings file, relative to . + /// + public const string StringsFile = @"strings\en-US.json"; + + /// + /// Path to the output folder + /// + /// + /// Can be an absolute path or a path relative to the binaries folder (bin\Debug). + /// Important: When changing the output path, make sure that "StringCheck.xslt" is present + /// to make the XML transform work. + /// + public const string OutputPath = @"."; + + /// + /// List of file extension to search. + /// + public static string[] TargetExtensions = new[] { "js", "html" }; + + /// + /// List of paths to exclude from search. + /// + public static string[] ExcludePaths = new[] { @"\bower_components\", @"\thirdparty\" }; + + private TestContext testContextInstance; + + /// + ///Gets or sets the test context which provides + ///information about and functionality for the current test run. + /// + public TestContext TestContext + { + get + { + return testContextInstance; + } + set + { + testContextInstance = value; + } + } + + [TestMethod] + public void ReportStringUsage() + { + this.CheckDashboardStrings(false); + } + + [TestMethod] + public void ReportUnusedStrings() + { + this.CheckDashboardStrings(true); + } + + private void CheckDashboardStrings(Boolean unusedOnly) + { + // Init Folders + var currentDir = System.IO.Directory.GetCurrentDirectory(); + Trace("CurrentDir: {0}", currentDir); + + var rootFolderInfo = ResolveFolder(currentDir, WebFolder); + Trace("Web Root: {0}", rootFolderInfo.FullName); + + var outputFolderInfo = ResolveFolder(currentDir, OutputPath); + Trace("Output Path: {0}", outputFolderInfo.FullName); + + // Load Strings + var stringsFileName = Path.Combine(rootFolderInfo.FullName, StringsFile); + + if (!File.Exists(stringsFileName)) + { + throw new Exception(string.Format("Strings file not found: {0}", stringsFileName)); + } + + int lineNumbers; + var stringsDic = this.CreateStringsDictionary(new FileInfo(stringsFileName), out lineNumbers); + + Trace("Loaded {0} strings from strings file containing {1} lines", stringsDic.Count, lineNumbers); + + var allFiles = rootFolderInfo.GetFiles("*", SearchOption.AllDirectories); + + var filteredFiles1 = allFiles.Where(f => TargetExtensions.Any(e => f.Name.EndsWith(e))); + var filteredFiles2 = filteredFiles1.Where(f => !ExcludePaths.Any(p => f.FullName.Contains(p))); + + var selectedFiles = filteredFiles2.OrderBy(f => f.FullName).ToList(); + + var wordIndex = IndexBuilder.BuildIndexFromFiles(selectedFiles, rootFolderInfo.FullName); + + Trace("Created word index from {0} files containing {1} individual words", selectedFiles.Count, wordIndex.Keys.Count); + + var outputFileName = Path.Combine(outputFolderInfo.FullName, string.Format("StringCheck_{0:yyyyMMddHHmmss}.xml", DateTime.Now)); + var settings = new XmlWriterSettings + { + Indent = true, + Encoding = Encoding.UTF8, + WriteEndDocumentOnClose = true + }; + + Trace("Output file: {0}", outputFileName); + + using (XmlWriter writer = XmlWriter.Create(outputFileName, settings)) + { + writer.WriteStartDocument(true); + + // Write the Processing Instruction node. + string xslText = "type=\"text/xsl\" href=\"StringCheck.xslt\""; + writer.WriteProcessingInstruction("xml-stylesheet", xslText); + + writer.WriteStartElement("StringUsages"); + writer.WriteAttributeString("ReportTitle", unusedOnly ? "Unused Strings Report" : "String Usage Report"); + writer.WriteAttributeString("Mode", unusedOnly ? "UnusedOnly" : "All"); + + foreach (var kvp in stringsDic) + { + var occurences = wordIndex.Find(kvp.Key); + + if (occurences == null || !unusedOnly) + { + ////Trace("{0}: {1}", kvp.Key, kvp.Value); + writer.WriteStartElement("Dictionary"); + writer.WriteAttributeString("Token", kvp.Key); + writer.WriteAttributeString("Text", kvp.Value); + + if (occurences != null && !unusedOnly) + { + foreach (var occurence in occurences) + { + writer.WriteStartElement("Occurence"); + writer.WriteAttributeString("FileName", occurence.FileName); + writer.WriteAttributeString("FullPath", occurence.FullPath); + writer.WriteAttributeString("LineNumber", occurence.LineNumber.ToString()); + writer.WriteEndElement(); + ////Trace(" {0}:{1}", occurence.FileName, occurence.LineNumber); + } + } + + writer.WriteEndElement(); + } + } + } + + TestContext.AddResultFile(outputFileName); + TestContext.AddResultFile(outputFolderInfo.FullName); + } + + private SortedDictionary CreateStringsDictionary(FileInfo file, out int lineNumbers) + { + var dic = new SortedDictionary(); + lineNumbers = 0; + + using (var reader = file.OpenText()) + { + while (!reader.EndOfStream) + { + lineNumbers++; + var words = reader + .ReadLine() + .Split(new[] { "\":" }, StringSplitOptions.RemoveEmptyEntries); + + + if (words.Length == 2) + { + var token = words[0].Replace("\"", string.Empty).Trim(); + var text = words[1].Replace("\",", string.Empty).Replace("\"", string.Empty).Trim(); + + if (dic.Keys.Contains(token)) + { + throw new Exception(string.Format("Double string entry found: {0}", token)); + } + + dic.Add(token, text); + } + } + } + + return dic; + } + + private DirectoryInfo ResolveFolder(string currentDir, string folderPath) + { + if (folderPath.IndexOf(@"\:") != 1) + { + folderPath = Path.Combine(currentDir, folderPath); + } + + var folderInfo = new DirectoryInfo(folderPath); + + if (!folderInfo.Exists) + { + throw new Exception(string.Format("Folder not found: {0}", folderInfo.FullName)); + } + + return folderInfo; + } + + + private void Trace(string message, params object[] parameters) + { + var formatted = string.Format(message, parameters); + System.Diagnostics.Trace.WriteLine(formatted); + } + } +} diff --git a/MediaBrowser.Tests/ConsistencyTests/TextIndexing/IndexBuilder.cs b/MediaBrowser.Tests/ConsistencyTests/TextIndexing/IndexBuilder.cs new file mode 100644 index 0000000000..07c0df86c7 --- /dev/null +++ b/MediaBrowser.Tests/ConsistencyTests/TextIndexing/IndexBuilder.cs @@ -0,0 +1,54 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace MediaBrowser.Tests.ConsistencyTests.TextIndexing +{ + public class IndexBuilder + { + public const int MinumumWordLength = 4; + + public static char[] WordChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890".ToCharArray(); + + public static WordIndex BuildIndexFromFiles(IEnumerable wordFiles, string rootFolderPath) + { + var index = new WordIndex(); + + var wordSeparators = Enumerable.Range(32, 127).Select(e => Convert.ToChar(e)).Where(c => !WordChars.Contains(c)).ToArray(); + wordSeparators = wordSeparators.Concat(new[] { '\t' }).ToArray(); // add tab + + foreach (var file in wordFiles) + { + var lineNumber = 1; + var displayFileName = file.FullName.Replace(rootFolderPath, string.Empty); + using (var reader = file.OpenText()) + { + while (!reader.EndOfStream) + { + var words = reader + .ReadLine() + .Split(wordSeparators, StringSplitOptions.RemoveEmptyEntries); + ////.Select(f => f.Trim()); + + var wordIndex = 1; + foreach (var word in words) + { + if (word.Length >= MinumumWordLength) + { + index.AddWordOccurrence(word, displayFileName, file.FullName, lineNumber, wordIndex++); + } + } + + lineNumber++; + } + } + } + + return index; + } + + } +} diff --git a/MediaBrowser.Tests/ConsistencyTests/TextIndexing/WordIndex.cs b/MediaBrowser.Tests/ConsistencyTests/TextIndexing/WordIndex.cs new file mode 100644 index 0000000000..4ced812373 --- /dev/null +++ b/MediaBrowser.Tests/ConsistencyTests/TextIndexing/WordIndex.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace MediaBrowser.Tests.ConsistencyTests.TextIndexing +{ + public class WordIndex : Dictionary + { + public WordIndex() : base(StringComparer.InvariantCultureIgnoreCase) + { + } + + public void AddWordOccurrence(string word, string fileName, string fullPath, int lineNumber, int wordIndex) + { + WordOccurrences current; + if (!this.TryGetValue(word, out current)) + { + current = new WordOccurrences(); + this[word] = current; + } + + current.AddOccurrence(fileName, fullPath, lineNumber, wordIndex); + } + + public WordOccurrences Find(string word) + { + WordOccurrences found; + if (this.TryGetValue(word, out found)) + { + return found; + } + + return null; + } + + } +} diff --git a/MediaBrowser.Tests/ConsistencyTests/TextIndexing/WordOccurrence.cs b/MediaBrowser.Tests/ConsistencyTests/TextIndexing/WordOccurrence.cs new file mode 100644 index 0000000000..40631f5825 --- /dev/null +++ b/MediaBrowser.Tests/ConsistencyTests/TextIndexing/WordOccurrence.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace MediaBrowser.Tests.ConsistencyTests.TextIndexing +{ + public struct WordOccurrence + { + public readonly string FileName; // file containing the word. + public readonly string FullPath; // file containing the word. + public readonly int LineNumber; // line within the file. + public readonly int WordIndex; // index within the line. + + public WordOccurrence(string fileName, string fullPath, int lineNumber, int wordIndex) + { + FileName = fileName; + FullPath = fullPath; + LineNumber = lineNumber; + WordIndex = wordIndex; + } + } +} diff --git a/MediaBrowser.Tests/ConsistencyTests/TextIndexing/WordOccurrences.cs b/MediaBrowser.Tests/ConsistencyTests/TextIndexing/WordOccurrences.cs new file mode 100644 index 0000000000..3ba3b59167 --- /dev/null +++ b/MediaBrowser.Tests/ConsistencyTests/TextIndexing/WordOccurrences.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace MediaBrowser.Tests.ConsistencyTests.TextIndexing +{ + public class WordOccurrences : List + { + public void AddOccurrence(string fileName, string fullPath, int lineNumber, int wordIndex) + { + this.Add(new WordOccurrence(fileName, fullPath, lineNumber, wordIndex)); + } + + } +} diff --git a/MediaBrowser.Tests/MediaBrowser.Tests.csproj b/MediaBrowser.Tests/MediaBrowser.Tests.csproj index 0cfe8182c8..76a1861097 100644 --- a/MediaBrowser.Tests/MediaBrowser.Tests.csproj +++ b/MediaBrowser.Tests/MediaBrowser.Tests.csproj @@ -25,6 +25,7 @@ DEBUG;TRACE prompt 4 + bin\Debug\MediaBrowser.Tests.XML none @@ -36,6 +37,7 @@ + @@ -50,6 +52,11 @@ + + + + + @@ -98,6 +105,14 @@ PreserveNewest + + + Always + StringCheck.xslt + + + + From f539c9894c91e15189b2d9d676afa8ea08b3f179 Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Tue, 26 Jul 2016 13:21:27 -0400 Subject: [PATCH 2/4] remove unused code --- MediaBrowser.WebDashboard/MediaBrowser.WebDashboard.csproj | 3 --- 1 file changed, 3 deletions(-) diff --git a/MediaBrowser.WebDashboard/MediaBrowser.WebDashboard.csproj b/MediaBrowser.WebDashboard/MediaBrowser.WebDashboard.csproj index e4150d85cf..7e4b39c364 100644 --- a/MediaBrowser.WebDashboard/MediaBrowser.WebDashboard.csproj +++ b/MediaBrowser.WebDashboard/MediaBrowser.WebDashboard.csproj @@ -173,9 +173,6 @@ PreserveNewest - - PreserveNewest - PreserveNewest From eb78381a3523985745da11931f5d2e08b2e6a0ab Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Wed, 27 Jul 2016 00:54:38 -0400 Subject: [PATCH 3/4] update resource loading --- MediaBrowser.WebDashboard/MediaBrowser.WebDashboard.csproj | 5 ----- 1 file changed, 5 deletions(-) diff --git a/MediaBrowser.WebDashboard/MediaBrowser.WebDashboard.csproj b/MediaBrowser.WebDashboard/MediaBrowser.WebDashboard.csproj index 7e4b39c364..5c520afcf2 100644 --- a/MediaBrowser.WebDashboard/MediaBrowser.WebDashboard.csproj +++ b/MediaBrowser.WebDashboard/MediaBrowser.WebDashboard.csproj @@ -1410,11 +1410,6 @@ PreserveNewest - - - PreserveNewest - - PreserveNewest From 2fed4c1ab8af48cfb6a0077189e6b8addbedb8d7 Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Wed, 27 Jul 2016 02:24:59 -0400 Subject: [PATCH 4/4] keep season/episode info up to date --- .../TV/EpisodeMetadataService.cs | 33 ++++++++++++++++++- .../TV/SeasonMetadataService.cs | 15 +++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/MediaBrowser.Providers/TV/EpisodeMetadataService.cs b/MediaBrowser.Providers/TV/EpisodeMetadataService.cs index a15de48660..b51b113805 100644 --- a/MediaBrowser.Providers/TV/EpisodeMetadataService.cs +++ b/MediaBrowser.Providers/TV/EpisodeMetadataService.cs @@ -1,4 +1,5 @@ -using MediaBrowser.Controller.Configuration; +using System; +using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities.TV; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Providers; @@ -6,12 +7,42 @@ using MediaBrowser.Model.Entities; using MediaBrowser.Model.Logging; using MediaBrowser.Providers.Manager; using System.Collections.Generic; +using System.Threading.Tasks; using CommonIO; namespace MediaBrowser.Providers.TV { public class EpisodeMetadataService : MetadataService { + protected override async Task BeforeSave(Episode item, bool isFullRefresh, ItemUpdateType currentUpdateType) + { + var updateType = await base.BeforeSave(item, isFullRefresh, currentUpdateType).ConfigureAwait(false); + + if (updateType <= ItemUpdateType.None) + { + if (!string.Equals(item.SeriesName, item.FindSeriesName(), StringComparison.Ordinal)) + { + updateType |= ItemUpdateType.MetadataImport; + } + } + if (updateType <= ItemUpdateType.None) + { + if (!string.Equals(item.SeriesSortName, item.FindSeriesSortName(), StringComparison.Ordinal)) + { + updateType |= ItemUpdateType.MetadataImport; + } + } + if (updateType <= ItemUpdateType.None) + { + if (!string.Equals(item.SeasonName, item.FindSeasonName(), StringComparison.Ordinal)) + { + updateType |= ItemUpdateType.MetadataImport; + } + } + + return updateType; + } + protected override void MergeData(MetadataResult source, MetadataResult target, List lockedFields, bool replaceData, bool mergeMetadataSettings) { ProviderUtils.MergeBaseItemData(source, target, lockedFields, replaceData, mergeMetadataSettings); diff --git a/MediaBrowser.Providers/TV/SeasonMetadataService.cs b/MediaBrowser.Providers/TV/SeasonMetadataService.cs index e4894915d6..f3e6f8e9c5 100644 --- a/MediaBrowser.Providers/TV/SeasonMetadataService.cs +++ b/MediaBrowser.Providers/TV/SeasonMetadataService.cs @@ -35,6 +35,21 @@ namespace MediaBrowser.Providers.TV updateType |= SaveIsVirtualItem(item, episodes); } + if (updateType <= ItemUpdateType.None) + { + if (!string.Equals(item.SeriesName, item.FindSeriesName(), StringComparison.Ordinal)) + { + updateType |= ItemUpdateType.MetadataImport; + } + } + if (updateType <= ItemUpdateType.None) + { + if (!string.Equals(item.SeriesSortName, item.FindSeriesSortName(), StringComparison.Ordinal)) + { + updateType |= ItemUpdateType.MetadataImport; + } + } + return updateType; }