LT-21804 Move PCPATR with FLEx into the FLEx repo

.gitattributes

@@ -31,3 +31,5 @@ Bin/ilrepack-assemblies -whitespace
 *.json -whitespace
 *.js -whitespace
 Src/LexText/ParserCore/ParserCoreTests/**/*.txt -whitespace
+Src/Utilities/pcpatrflex/DisambiguateInFLExDB/DisambiguateInFLExDBTests/TestData/*.* -whitespace
+Src/Utilities/pcpatrflex/PcPatrBrowserDll/Transforms/*.* -whitespace

.gitignore

@@ -143,6 +143,7 @@ DistFiles/ReleaseData/
 # Exceptions
 !Lib/Windows/Debug
 !Lib/Windows/Release
+!Src/Utilities/pcpatrflex/DisambiguateInFLExDB/DisambiguateInFLExDBTests/TestData/*.log
 
 Lib/icu*.lib
 DistFiles/Icu*/data

Build/PackageRestore.targets

@@ -30,6 +30,8 @@
 		<DownloadsDir>$(fwrt)/Downloads</DownloadsDir>
 		<!-- GeckofxHtmlToPdf: download from GitHub release (https://github.com/sillsdev/geckofxHtmlToPdf/releases) -->
 		<GeckofxHtmlToPdfVersion>1.1.0</GeckofxHtmlToPdfVersion>
+		<!-- pc-parse: download from GitHub release (https://github.com/sillsdev/CarlaLegacy/releases/) -->
+		<PcParseVersion>pc-parse1.0.0</PcParseVersion>
 		<!-- PackagesDir: Use local packages folder. -->
 		<PackagesDir>$(fwrt)/packages</PackagesDir>
 	</PropertyGroup>
@@ -282,6 +284,19 @@
 			DownloadsDir="$(DownloadsDir)"
 			Condition="'$(OS)'=='Windows_NT' AND !Exists('$(DownloadsDir)/Args.dll')"
 		/>
+		<!-- pc-parse binaries downloaded from GitHub release. -->
+		<DownloadFile
+			Address="https://github.com/sillsdev/CarlaLegacy/releases/download/$(PcParseVersion)/pcpatr64.exe"
+			LocalFilename="pcpatr64.exe"
+			DownloadsDir="$(DownloadsDir)"
+			Condition="'$(OS)'=='Windows_NT' AND !Exists('$(DownloadsDir)/pcpatr64.exe')"
+		/>
+		<DownloadFile
+			Address="https://github.com/sillsdev/CarlaLegacy/releases/download/$(PcParseVersion)/TonePars64.exe"
+			LocalFilename="TonePars64.exe"
+			DownloadsDir="$(DownloadsDir)"
+			Condition="'$(OS)'=='Windows_NT' AND !Exists('$(DownloadsDir)/TonePars64.exe')"
+		/>
 	</Target>
 
 	<!-- ======================================================================================== -->
@@ -527,6 +542,21 @@
 			OverwriteReadOnlyFiles="true"
 			Condition="Exists('$(DownloadsDir)/GeckofxHtmlToPdf.exe.config')"
 		/>
+		<!-- Copy pre-populated binaries from pc-parse -->
+		<Copy
+			SourceFiles="$(DownloadsDir)/pcpatr64.exe"
+			DestinationFiles="$(dir-outputBase)/pcpatr64.exe"
+			SkipUnchangedFiles="true"
+			OverwriteReadOnlyFiles="true"
+			Condition="Exists('$(DownloadsDir)/pcpatr64.exe')"
+		/>
+		<Copy
+			SourceFiles="$(DownloadsDir)/TonePars64.exe"
+			DestinationFiles="$(dir-outputBase)/TonePars64.exe"
+			SkipUnchangedFiles="true"
+			OverwriteReadOnlyFiles="true"
+			Condition="Exists('$(DownloadsDir)/TonePars64.exe')"
+		/>
 		<!-- Encoding Converters Files (x64 only) -->
 		<!-- Note: encoding-converters-core package doesn't follow standard NuGet runtime conventions
 		     (has extra EcDistFiles folder), so automatic runtime asset copying doesn't work.

DistFiles/Language Explorer/Configuration/Grammar/ToneParscd.tab

@@ -0,0 +1,12 @@
+\unified \lx
+\ch '\u' 'U'
+\ch '\eType' 'T'
+\ch '\a' 'A'
+\ch '\c' 'C'
+\ch '\wc' 'F'
+\ch '\o' 'O'
+\ch '\lx' 'M'
+\ch '\mp' 'P'
+\ch '\mcc' 'Z'
+\ch '\loc' 'L'
+\ch '\noload' '!'

DistFiles/Language Explorer/Configuration/Grammar/XAmplecd.tab

@@ -0,0 +1,13 @@
+\unified \lx
+\ch '\u' 'U'
+\ch '\eType' 'T'
+\ch '\a' 'A'
+\ch '\c' 'C'
+\ch '\wc' 'c'
+\ch '\o' 'O'
+\ch '\lx' 'M'
+\ch '\fd' 'F'
+\ch '\mp' 'P'
+\ch '\mcc' 'Z'
+\ch '\loc' 'L'
+\ch '\noload' '!'

FieldWorks.sln

@@ -258,6 +258,24 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "VarGenServiceTests", "Src\U
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "VariantGeneratorDll", "Src\Utilities\AlloVarGen\VariantGeneratorDll\VariantGeneratorDll.csproj", "{91219E68-FF1D-4DED-BB06-3A6AF46C0419}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DisambiguateInFLExDB", "Src\Utilities\pcpatrflex\DisambiguateInFLExDB\DisambiguateInFLExDB.csproj", "{6A3359E7-E3DA-4CF6-B6F8-C6A4E8D9800B}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ExtractAnaFilesFromTexts", "Src\Utilities\pcpatrflex\ExtractAnaFilesFromTexts\ExtractAnaFilesFromTexts.csproj", "{F9AC363B-9EDF-4A01-BBB8-79810AB208E4}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PrepFLExDBDll", "Src\Utilities\pcpatrflex\PrepFLExDBDll\PrepFLExDBDll.csproj", "{7B58967B-E363-40E9-ACC7-80F99FC1A50C}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PrepFLExDBTests", "Src\Utilities\pcpatrflex\PrepFLExDBDll\PrepFLExDBTests\PrepFLExDBTests.csproj", "{FB6117BB-990E-4493-A7FF-430E2CCE61BD}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PcPatrBrowserDll", "Src\Utilities\pcpatrflex\PcPatrBrowserDll\PcPatrBrowserDll.csproj", "{77CBCCAA-F488-4533-BA39-C676BDDF292D}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LingTreeTree", "Src\Utilities\pcpatrflex\LingTreeTree\LingTreeTree.csproj", "{102FA930-E921-4B94-BDA6-9D4F16D829AE}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PcPatrFLExDll", "Src\Utilities\pcpatrflex\PcPatrFLExDll\PcPatrFLExDll.csproj", "{0B09C5E2-7DD0-46AD-ADA9-00040CDE6F7E}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ToneParsFLExDll", "Src\Utilities\pcpatrflex\ToneParsFLExDll\ToneParsFLExDll.csproj", "{FE322FAD-5208-4FB3-8E71-C8C09CDC3575}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DisambiguateInFLExDBTests", "Src\Utilities\pcpatrflex\DisambiguateInFLExDB\DisambiguateInFLExDBTests\DisambiguateInFLExDBTests.csproj", "{1D9F7F7D-F4DE-43DC-9E1D-9D0E512D1CB6}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Bounds|x64 = Bounds|x64
@@ -973,6 +991,60 @@ Global
 		{5DB1CEDA-B7AA-4594-9CE0-6D3A6F5763DF}.Debug|x64.Build.0 = Debug|x64
 		{5DB1CEDA-B7AA-4594-9CE0-6D3A6F5763DF}.Release|x64.ActiveCfg = Release|x64
 		{5DB1CEDA-B7AA-4594-9CE0-6D3A6F5763DF}.Release|x64.Build.0 = Release|x64
+		{6A3359E7-E3DA-4CF6-B6F8-C6A4E8D9800B}.Bounds|x64.ActiveCfg = Debug|x64
+		{6A3359E7-E3DA-4CF6-B6F8-C6A4E8D9800B}.Bounds|x64.Build.0 = Debug|x64
+		{6A3359E7-E3DA-4CF6-B6F8-C6A4E8D9800B}.Debug|x64.ActiveCfg = Debug|x64
+		{6A3359E7-E3DA-4CF6-B6F8-C6A4E8D9800B}.Debug|x64.Build.0 = Debug|x64
+		{6A3359E7-E3DA-4CF6-B6F8-C6A4E8D9800B}.Release|x64.ActiveCfg = Release|x64
+		{6A3359E7-E3DA-4CF6-B6F8-C6A4E8D9800B}.Release|x64.Build.0 = Release|x64
+		{F9AC363B-9EDF-4A01-BBB8-79810AB208E4}.Bounds|x64.ActiveCfg = Debug|x64
+		{F9AC363B-9EDF-4A01-BBB8-79810AB208E4}.Bounds|x64.Build.0 = Debug|x64
+		{F9AC363B-9EDF-4A01-BBB8-79810AB208E4}.Debug|x64.ActiveCfg = Debug|x64
+		{F9AC363B-9EDF-4A01-BBB8-79810AB208E4}.Debug|x64.Build.0 = Debug|x64
+		{F9AC363B-9EDF-4A01-BBB8-79810AB208E4}.Release|x64.ActiveCfg = Release|x64
+		{F9AC363B-9EDF-4A01-BBB8-79810AB208E4}.Release|x64.Build.0 = Release|x64
+		{7B58967B-E363-40E9-ACC7-80F99FC1A50C}.Bounds|x64.ActiveCfg = Debug|x64
+		{7B58967B-E363-40E9-ACC7-80F99FC1A50C}.Bounds|x64.Build.0 = Debug|x64
+		{7B58967B-E363-40E9-ACC7-80F99FC1A50C}.Debug|x64.ActiveCfg = Debug|x64
+		{7B58967B-E363-40E9-ACC7-80F99FC1A50C}.Debug|x64.Build.0 = Debug|x64
+		{7B58967B-E363-40E9-ACC7-80F99FC1A50C}.Release|x64.ActiveCfg = Release|x64
+		{7B58967B-E363-40E9-ACC7-80F99FC1A50C}.Release|x64.Build.0 = Release|x64
+		{FB6117BB-990E-4493-A7FF-430E2CCE61BD}.Bounds|x64.ActiveCfg = Debug|x64
+		{FB6117BB-990E-4493-A7FF-430E2CCE61BD}.Bounds|x64.Build.0 = Debug|x64
+		{FB6117BB-990E-4493-A7FF-430E2CCE61BD}.Debug|x64.ActiveCfg = Debug|x64
+		{FB6117BB-990E-4493-A7FF-430E2CCE61BD}.Debug|x64.Build.0 = Debug|x64
+		{FB6117BB-990E-4493-A7FF-430E2CCE61BD}.Release|x64.ActiveCfg = Release|x64
+		{FB6117BB-990E-4493-A7FF-430E2CCE61BD}.Release|x64.Build.0 = Release|x64
+		{77CBCCAA-F488-4533-BA39-C676BDDF292D}.Bounds|x64.ActiveCfg = Debug|x64
+		{77CBCCAA-F488-4533-BA39-C676BDDF292D}.Bounds|x64.Build.0 = Debug|x64
+		{77CBCCAA-F488-4533-BA39-C676BDDF292D}.Debug|x64.ActiveCfg = Debug|x64
+		{77CBCCAA-F488-4533-BA39-C676BDDF292D}.Debug|x64.Build.0 = Debug|x64
+		{77CBCCAA-F488-4533-BA39-C676BDDF292D}.Release|x64.ActiveCfg = Release|x64
+		{77CBCCAA-F488-4533-BA39-C676BDDF292D}.Release|x64.Build.0 = Release|x64
+		{102FA930-E921-4B94-BDA6-9D4F16D829AE}.Bounds|x64.ActiveCfg = Debug|x64
+		{102FA930-E921-4B94-BDA6-9D4F16D829AE}.Bounds|x64.Build.0 = Debug|x64
+		{102FA930-E921-4B94-BDA6-9D4F16D829AE}.Debug|x64.ActiveCfg = Debug|x64
+		{102FA930-E921-4B94-BDA6-9D4F16D829AE}.Debug|x64.Build.0 = Debug|x64
+		{102FA930-E921-4B94-BDA6-9D4F16D829AE}.Release|x64.ActiveCfg = Release|x64
+		{102FA930-E921-4B94-BDA6-9D4F16D829AE}.Release|x64.Build.0 = Release|x64
+		{0B09C5E2-7DD0-46AD-ADA9-00040CDE6F7E}.Bounds|x64.ActiveCfg = Debug|x64
+		{0B09C5E2-7DD0-46AD-ADA9-00040CDE6F7E}.Bounds|x64.Build.0 = Debug|x64
+		{0B09C5E2-7DD0-46AD-ADA9-00040CDE6F7E}.Debug|x64.ActiveCfg = Debug|x64
+		{0B09C5E2-7DD0-46AD-ADA9-00040CDE6F7E}.Debug|x64.Build.0 = Debug|x64
+		{0B09C5E2-7DD0-46AD-ADA9-00040CDE6F7E}.Release|x64.ActiveCfg = Release|x64
+		{0B09C5E2-7DD0-46AD-ADA9-00040CDE6F7E}.Release|x64.Build.0 = Release|x64
+		{FE322FAD-5208-4FB3-8E71-C8C09CDC3575}.Bounds|x64.ActiveCfg = Debug|x64
+		{FE322FAD-5208-4FB3-8E71-C8C09CDC3575}.Bounds|x64.Build.0 = Debug|x64
+		{FE322FAD-5208-4FB3-8E71-C8C09CDC3575}.Debug|x64.ActiveCfg = Debug|x64
+		{FE322FAD-5208-4FB3-8E71-C8C09CDC3575}.Debug|x64.Build.0 = Debug|x64
+		{FE322FAD-5208-4FB3-8E71-C8C09CDC3575}.Release|x64.ActiveCfg = Release|x64
+		{FE322FAD-5208-4FB3-8E71-C8C09CDC3575}.Release|x64.Build.0 = Release|x64
+		{1D9F7F7D-F4DE-43DC-9E1D-9D0E512D1CB6}.Bounds|x64.ActiveCfg = Debug|x64
+		{1D9F7F7D-F4DE-43DC-9E1D-9D0E512D1CB6}.Bounds|x64.Build.0 = Debug|x64
+		{1D9F7F7D-F4DE-43DC-9E1D-9D0E512D1CB6}.Debug|x64.ActiveCfg = Debug|x64
+		{1D9F7F7D-F4DE-43DC-9E1D-9D0E512D1CB6}.Debug|x64.Build.0 = Debug|x64
+		{1D9F7F7D-F4DE-43DC-9E1D-9D0E512D1CB6}.Release|x64.ActiveCfg = Release|x64
+		{1D9F7F7D-F4DE-43DC-9E1D-9D0E512D1CB6}.Release|x64.Build.0 = Release|x64
 		{996498A3-06F1-4B1B-B83F-15648DD8F514}.Bounds|x64.ActiveCfg = Debug|x64
 		{996498A3-06F1-4B1B-B83F-15648DD8F514}.Bounds|x64.Build.0 = Debug|x64
 		{996498A3-06F1-4B1B-B83F-15648DD8F514}.Debug|x64.ActiveCfg = Debug|x64

Src/Utilities/pcpatrflex/DisambiguateInFLExDB/AndFileLoader.cs

@@ -0,0 +1,68 @@
+// Copyright (c) 2018 SIL International
+// This software is licensed under the LGPL, version 2.1 or later
+// (http://www.gnu.org/licenses/lgpl-2.1.html)
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace SIL.DisambiguateInFLExDB
+{
+	public static class AndFileLoader
+	{
+		static string[] SplitOn = new string[] { "\\parse", "\\endparse" };
+		static string[] pField = new string[] { "\\p " };
+
+		public static string[] GetGuidsFromAndFile(string andfile)
+		{
+			var guids = new List<string>();
+			using (var sr = new StreamReader(andfile, Encoding.UTF8))
+			{
+				var contents = sr.ReadToEnd();
+				sr.Close();
+				// skip any \id fields at the beginning
+				int iStart = Math.Max(0, contents.IndexOf(pField[0]));
+				var sections = contents
+					.Substring(iStart)
+					.Split(SplitOn, StringSplitOptions.RemoveEmptyEntries);
+				foreach (string section in sections)
+				{
+					if (section.Contains("\\p"))
+					{
+						ProcessAna(guids, section);
+					}
+				}
+			}
+			return guids.ToArray();
+		}
+
+		private static void ProcessAna(List<string> guids, string ana)
+		{
+			if (ana.Contains("%"))
+			{
+				// still ambiguous; skip it
+				guids.Add("");
+			}
+			else
+			{
+				var ps = ana.Split(pField, StringSplitOptions.RemoveEmptyEntries);
+				var sb = new StringBuilder();
+				foreach (string p in ps)
+				{
+					if (p.Length >= 2 && p[0] != '\r' && p[1] != '\n')
+					{
+						int i = p.IndexOf("\n");
+						string start = p.Substring(0, i + 1);
+						var clean = start.Replace("\r", "").Replace("\n", "");
+						sb.Append(clean);
+						sb.Append("\n");
+					}
+				}
+				guids.Add(sb.ToString());
+			}
+		}
+	}
+}

Src/Utilities/pcpatrflex/DisambiguateInFLExDB/DisambiguateInFLExDB.csproj

@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <AssemblyName>SIL.DisambiguateInFLExDB</AssemblyName>
+    <RootNamespace>SIL.DisambiguateInFLExDB</RootNamespace>
+    <TargetFramework>net48</TargetFramework>
+    <OutputType>Library</OutputType>    <NoWarn>168,169,219,414,649,1635,1702,1701</NoWarn>
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <Prefer32Bit>false</Prefer32Bit>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <DebugSymbols>true</DebugSymbols>
+    <Optimize>false</Optimize>
+    <DebugType>portable</DebugType>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
+    <DefineConstants>TRACE</DefineConstants>
+    <DebugSymbols>true</DebugSymbols>
+    <Optimize>true</Optimize>
+    <DebugType>portable</DebugType>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" />
+    <PackageReference Include="SIL.Core" />
+    <PackageReference Include="SIL.LCModel" />
+    <PackageReference Include="SIL.LCModel.Core" />
+    <PackageReference Include="System.ValueTuple" />
+  </ItemGroup>
+  <ItemGroup>
+    <Reference Include="netstandard" />
+    <Reference Include="System.Xml.Linq" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="../../../Common/FwUtils/FwUtils.csproj" />
+    <ProjectReference Include="../../../LexText/ParserCore/ParserCore.csproj" />
+    <ProjectReference Include="../../../LexText/ParserCore/XAmpleManagedWrapper/XAmpleManagedWrapper.csproj" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Remove="DisambiguateInFLExDBTests\**" />
+    <EmbeddedResource Remove="DisambiguateInFLExDBTests\**" />
+    <None Remove="DisambiguateInFLExDBTests\**" />
+    <Compile Include="..\..\..\CommonAssemblyInfo.cs">
+      <Link>Properties\CommonAssemblyInfo.cs</Link>
+    </Compile>
+  </ItemGroup>
+</Project>

Src/Utilities/pcpatrflex/DisambiguateInFLExDB/DisambiguateInFLExDBTests/AndFileLoaderTest.cs

@@ -0,0 +1,55 @@
+// Copyright (c) 2018 SIL International
+// This software is licensed under the LGPL, version 2.1 or later
+// (http://www.gnu.org/licenses/lgpl-2.1.html)
+
+using NUnit.Framework;
+using SIL.DisambiguateInFLExDB;
+using SIL.DisambiguateInFLExDBTests;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace SIL.DisambiguateInFLExDBTest
+{
+	[TestFixture]
+	class AndFileLoaderTest : DisambiguateTests
+	{
+		string AndFile { get; set; }
+
+		[Test]
+		public void LoadAndFileGuidsTest()
+		{
+			AndFile = Path.Combine(TestDataDir, "Text4LoadTest.and");
+			var result = AndFileLoader.GetGuidsFromAndFile(AndFile);
+			Assert.AreEqual(17, result.Length);
+			var guid = result[0];
+			Assert.AreEqual(
+				"d3f40ba4-6fc3-4696-a648-cc6d51f6fe1e\n399dd27b-5ff1-4719-87ac-58b2f0178463\n7df77c51-3b54-44a5-9a2c-fac71c31a197\n4172bab0-cbf7-4e8f-96b5-1ac409c99276=91a77900-98fe-45f1-ae54-7886db43d64e\n",
+				guid.ToString()
+			);
+			guid = result[1];
+			Assert.AreEqual("", guid.ToString());
+			guid = result[3];
+			Assert.AreEqual(
+				"7e6c1be9-ad21-454e-91b4-dfc126514da7\n39994213-303e-4e59-9e47-f10e74d7331d\n1ea23f59-f6d9-406d-89f6-792318a04efe\n",
+				guid.ToString()
+			);
+			guid = result[4];
+			Assert.AreEqual(
+				"e2e4949d-9af0-4142-9d4f-f2d9afdcb646\nb3e8623e-5679-4261-acd5-d62ed71d1d2b\n9be2d38f-bc3a-4e96-acb5-64d2b3e53d95\n0dee3420-0d8e-4506-8737-c5a78b85188a\n1ea23f59-f6d9-406d-89f6-792318a04efe\n479aca02-ca6a-4c2a-862a-d980fbcc9a37\n04f021dc-a0dd-44fc-8b0a-9e6741743dd8\n07fbf262-bbe7-415b-af3f-8317a2cb4521\n",
+				guid.ToString()
+			);
+			guid = result[6];
+			Assert.AreEqual("", guid.ToString());
+			guid = result[16];
+			Assert.AreEqual(
+				"e2e4949d-9af0-4142-9d4f-f2d9afdcb646\n216db198-8a9e-43e6-ba00-f10db3d51465\n9be2d38f-bc3a-4e96-acb5-64d2b3e53d95\n0053c955-c19a-480b-bb45-9a1b27b3d5eb\n933d7fb3-b038-4913-b5b8-576f80df2fba\n7841d0ff-57f0-4a2c-a689-6d109efca66e\nb8c624a0-9fb3-4e39-867d-802292933ed5\n",
+				guid.ToString()
+			);
+		}
+	}
+}