Fix NativeAOT assembly name mangling collisions

src/coreclr/tools/Common/Compiler/NativeAotNameMangler.cs

@@ -129,6 +129,7 @@ private static Utf8String SanitizeNameWithHash(Utf8String literal, byte[] hash =
         /// Dictionary given a mangled name for a given <see cref="TypeDesc"/>
         /// </summary>
         private Dictionary<TypeDesc, Utf8String> _mangledTypeNames = new Dictionary<TypeDesc, Utf8String>();
+        private Dictionary<string, Utf8String> _mangledAssemblyNames = new Dictionary<string, Utf8String>(StringComparer.Ordinal);
 
         /// <summary>
         /// Given a set of names <param name="set"/> check if <param name="origName"/>
@@ -195,6 +196,75 @@ protected Utf8String NestMangledName(Utf8String name)
             return Utf8String.Concat(EnterNameScopeSequence, name, ExitNameScopeSequence);
         }
 
+        private Utf8String GetMangledAssemblyName(EcmaAssembly assembly)
+        {
+            string assemblyName = assembly.GetName().Name;
+            lock (this)
+            {
+                if (_mangledAssemblyNames.TryGetValue(assemblyName, out Utf8String mangledName))
+                    return mangledName;
+
+                return ComputeMangledAssemblyName(assembly);
+            }
+        }
+
+        private Utf8String ComputeMangledAssemblyName(EcmaAssembly assembly)
+        {
+            lock (this)
+            {
+                string assemblyName = assembly.GetName().Name;
+                if (!_mangledAssemblyNames.TryGetValue(assemblyName, out Utf8String name))
+                {
+                    CompilerTypeSystemContext context = (CompilerTypeSystemContext)assembly.Context;
+                    var assemblyNames = new List<string>(context.InputFilePaths.Count + context.ReferenceFilePaths.Count);
+                    assemblyNames.AddRange(context.InputFilePaths.Keys);
+                    assemblyNames.AddRange(context.ReferenceFilePaths.Keys);
+                    assemblyNames.Sort(CompareAssemblyNamesForMangling);
+
+                    var deduplicator = new HashSet<Utf8String>();
+                    foreach (string candidateAssemblyName in assemblyNames)
+                    {
+                        if (_mangledAssemblyNames.TryGetValue(candidateAssemblyName, out Utf8String existingMangledName))
+                        {
+                            deduplicator.Add(existingMangledName);
+                            continue;
+                        }
+
+                        bool isSystemPrivate = IsSystemPrivateAssemblyName(candidateAssemblyName);
+                        string prefixAssemblyName = isSystemPrivate
+                            ? string.Concat("S.P.", candidateAssemblyName.AsSpan(15))
+                            : candidateAssemblyName;
+
+                        name = SanitizeName(new Utf8String(prefixAssemblyName));
+
+                        if (!isSystemPrivate)
+                            name = DisambiguateName(name, deduplicator);
+
+                        deduplicator.Add(name);
+                        _mangledAssemblyNames.Add(candidateAssemblyName, name);
+                    }
+
+                    name = _mangledAssemblyNames[assemblyName];
+                }
+                return name;
+            }
+
+            static int CompareAssemblyNamesForMangling(string left, string right)
+            {
+                bool leftSystemPrivate = IsSystemPrivateAssemblyName(left);
+                bool rightSystemPrivate = IsSystemPrivateAssemblyName(right);
+                if (leftSystemPrivate != rightSystemPrivate)
+                    return leftSystemPrivate ? -1 : 1;
+
+                return string.CompareOrdinal(left, right);
+            }
+
+            static bool IsSystemPrivateAssemblyName(string assemblyName)
+            {
+                return assemblyName.StartsWith("System.Private.", StringComparison.Ordinal);
+            }
+        }
+
         /// <summary>
         /// If given <param name="type"/> is an <see cref="EcmaType"/> precompute its mangled type name
         /// along with all the other types from the same module as <param name="type"/>.
@@ -215,15 +285,7 @@ private Utf8String ComputeMangledTypeName(TypeDesc type)
                     {
                         bool isSystemModule = ecmaType.Module == ecmaType.Context.SystemModule;
 
-                        string assemblyName = ((EcmaAssembly)ecmaType.Module).GetName().Name;
-                        bool isSystemPrivate = assemblyName.StartsWith("System.Private.");
-
-                        // Abbreviate System.Private to S.P. This might conflict with user defined assembly names,
-                        // but we already have a problem due to running SanitizeName without disambiguating the result
-                        // This problem needs a better fix.
-                        if (isSystemPrivate)
-                            assemblyName = string.Concat("S.P.", assemblyName.AsSpan(15));
-                        Utf8String prependAssemblyName = SanitizeName(new Utf8String(assemblyName));
+                        Utf8String prependAssemblyName = GetMangledAssemblyName((EcmaAssembly)ecmaType.Module);
 
                         var deduplicator = new HashSet<Utf8String>();
 

src/coreclr/tools/aot/ILCompiler.Compiler.Tests/ILCompiler.Compiler.Tests.Assets/ILCompiler.Compiler.Tests.Assets.csproj

@@ -12,6 +12,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <Compile Remove="ManglerCollision.*\**" />
     <ProjectReference Include="..\..\..\..\nativeaot\Test.CoreLib\src\Test.CoreLib.csproj" />
   </ItemGroup>
 

src/coreclr/tools/aot/ILCompiler.Compiler.Tests/ILCompiler.Compiler.Tests.Assets/ManglerCollision.A.B/ManglerCollision.A.B.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Library</OutputType>
+    <AssemblyName>A.B</AssemblyName>
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <GenerateTargetFrameworkAttribute>false</GenerateTargetFrameworkAttribute>
+    <SkipTestRun>true</SkipTestRun>
+    <TargetFramework>$(NetCoreAppToolCurrent)</TargetFramework>
+    <DisableImplicitFrameworkReferences>true</DisableImplicitFrameworkReferences>
+    <RunAnalyzers>false</RunAnalyzers>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\..\..\nativeaot\Test.CoreLib\src\Test.CoreLib.csproj" />
+  </ItemGroup>
+</Project>

src/coreclr/tools/aot/ILCompiler.Compiler.Tests/ILCompiler.Compiler.Tests.Assets/ManglerCollision.A.B/TestType.cs

@@ -0,0 +1,9 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace ManglerCollision
+{
+    public class TestType
+    {
+    }
+}

src/coreclr/tools/aot/ILCompiler.Compiler.Tests/ILCompiler.Compiler.Tests.Assets/ManglerCollision.A_B/ManglerCollision.A_B.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Library</OutputType>
+    <AssemblyName>A_B</AssemblyName>
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <GenerateTargetFrameworkAttribute>false</GenerateTargetFrameworkAttribute>
+    <SkipTestRun>true</SkipTestRun>
+    <TargetFramework>$(NetCoreAppToolCurrent)</TargetFramework>
+    <DisableImplicitFrameworkReferences>true</DisableImplicitFrameworkReferences>
+    <RunAnalyzers>false</RunAnalyzers>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\..\..\nativeaot\Test.CoreLib\src\Test.CoreLib.csproj" />
+  </ItemGroup>
+</Project>

src/coreclr/tools/aot/ILCompiler.Compiler.Tests/ILCompiler.Compiler.Tests.Assets/ManglerCollision.A_B/TestType.cs

@@ -0,0 +1,9 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace ManglerCollision
+{
+    public class TestType
+    {
+    }
+}

src/coreclr/tools/aot/ILCompiler.Compiler.Tests/ILCompiler.Compiler.Tests.csproj

@@ -37,11 +37,22 @@
       <OutputItemType>Content</OutputItemType>
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </ProjectReference>
+    <ProjectReference Include="ILCompiler.Compiler.Tests.Assets\ManglerCollision.A.B\ManglerCollision.A.B.csproj">
+      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+      <OutputItemType>Content</OutputItemType>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </ProjectReference>
+    <ProjectReference Include="ILCompiler.Compiler.Tests.Assets\ManglerCollision.A_B\ManglerCollision.A_B.csproj">
+      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+      <OutputItemType>Content</OutputItemType>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </ProjectReference>
   </ItemGroup>
 
   <ItemGroup>
     <Compile Include="DependencyGraphTests.cs" />
     <Compile Include="DevirtualizationTests.cs" />
+    <Compile Include="NameManglerTests.cs" />
     <Compile Include="SwiftLoweringTests.cs" />
   </ItemGroup>
 </Project>

src/coreclr/tools/aot/ILCompiler.Compiler.Tests/NameManglerTests.cs

@@ -0,0 +1,42 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+
+using Internal.IL;
+using Internal.TypeSystem;
+
+using Xunit;
+
+namespace ILCompiler.Compiler.Tests
+{
+    public class NameManglerTests
+    {
+        [Fact]
+        public void SanitizedAssemblyNameCollisionsAreDisambiguatedDeterministically()
+        {
+            var target = new TargetDetails(TargetArchitecture.X64, TargetOS.Windows, TargetAbi.NativeAot);
+            var context = new CompilerTypeSystemContext(target, SharedGenericsMode.CanonicalReferenceTypes, DelegateFeature.All);
+
+            context.InputFilePaths = new Dictionary<string, string> {
+                { "Test.CoreLib", @"Test.CoreLib.dll" },
+                { "A_B", @"A_B.dll" },
+                { "A.B", @"A.B.dll" },
+                };
+            context.ReferenceFilePaths = new Dictionary<string, string>();
+
+            context.SetSystemModule(context.GetModuleForSimpleName("Test.CoreLib"));
+
+            ModuleDesc moduleWithDot = context.GetModuleForSimpleName("A.B");
+            ModuleDesc moduleWithUnderscore = context.GetModuleForSimpleName("A_B");
+
+            var nameMangler = new NativeAotNameMangler(new UnixNodeMangler());
+
+            string typeNameWithUnderscoreAssembly = nameMangler.GetMangledTypeName(moduleWithUnderscore.GetType("ManglerCollision"u8, "TestType"u8)).ToString();
+            string typeNameWithDotAssembly = nameMangler.GetMangledTypeName(moduleWithDot.GetType("ManglerCollision"u8, "TestType"u8)).ToString();
+
+            Assert.Equal("A_B_ManglerCollision_TestType", typeNameWithDotAssembly);
+            Assert.Equal("A_B_0_ManglerCollision_TestType", typeNameWithUnderscoreAssembly);
+        }
+    }
+}