diff --git a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/language/AnalyzeText.scala b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/language/AnalyzeText.scala index 0cd1eb155a4..9c7128f767e 100644 --- a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/language/AnalyzeText.scala +++ b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/language/AnalyzeText.scala @@ -169,7 +169,7 @@ class AnalyzeText(override val uid: String) extends CognitiveServicesBase(uid) "SentimentAnalysis") setDefault( - apiVersion -> Left("2022-05-01") + apiVersion -> Left("2024-11-01") ) override def urlPath: String = "/language/:analyze-text" diff --git a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/translate/TextTranslator.scala b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/translate/TextTranslator.scala index 88dabdf63ae..4bc62b2ed45 100644 --- a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/translate/TextTranslator.scala +++ b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/translate/TextTranslator.scala @@ -9,8 +9,9 @@ import com.microsoft.azure.synapse.ml.io.http.SimpleHTTPTransformer import com.microsoft.azure.synapse.ml.logging.{FeatureNames, SynapseMLLogging} import com.microsoft.azure.synapse.ml.param.ServiceParam import com.microsoft.azure.synapse.ml.stages.{DropColumns, Lambda} -import org.apache.http.client.methods.{HttpPost, HttpRequestBase} +import org.apache.http.client.methods.{HttpGet, HttpPost, HttpRequestBase} import org.apache.http.entity.{AbstractHttpEntity, StringEntity} +import org.apache.spark.ml.param.{Param, StringArrayParam} import org.apache.spark.ml.util.Identifiable import org.apache.spark.ml.{ComplexParamsReadable, NamespaceInjections, PipelineModel, Transformer} import org.apache.spark.sql.Row @@ -77,7 +78,14 @@ trait HasToLanguage extends HasServiceParams { def getToLanguageCol: String = getVectorParam(toLanguage) } +private[translate] object TranslatorApiVersion { + val V3: String = "3.0" + val V2026: String = "2026-06-06" + val Supported: Set[String] = Set(V3, V2026) +} + trait TextAsOnlyEntity extends HasTextInput with HasCognitiveServiceInput with HasSubscriptionRegion { + this: TextTranslatorBase => override protected def contentType: Row => String = { _ => "application/json; charset=UTF-8" } @@ -85,37 +93,47 @@ trait TextAsOnlyEntity extends HasTextInput with HasCognitiveServiceInput with H { row: Row => if (shouldSkip(row)) { None - } else if (getValue(row, text).forall(Option(_).isEmpty)) { - None } else { val urlParams: Array[ServiceParam[Any]] = getUrlParams.asInstanceOf[Array[ServiceParam[Any]]] + val version = getApiVersion + validateApiVersion(version) val texts = getValue(row, text) - - val base = getUrl + "?api-version=3.0" - val appended = if (!urlParams.isEmpty) { - "&" + URLEncodingUtils.format(urlParams.flatMap(p => - getValueOpt(row, p).map { - val pName = p.name match { - case "fromLanguage" => "from" - case "toLanguage" => "to" - case s => s - } - v => pName -> p.toValueString(v) - } - ).toMap) + .flatMap(Option(_)) + .filter(value => version == TranslatorApiVersion.V3 || value.trim.nonEmpty) + if (texts.isEmpty) { + None } else { - "" - } + val base = getUrl + s"?api-version=$version" + val appended = if (!urlParams.isEmpty) { + "&" + URLEncodingUtils.format(urlParams.flatMap(p => + getValueOpt(row, p).map { + val pName = p.name match { + case "fromLanguage" => "from" + case "toLanguage" => "to" + case s => s + } + v => pName -> p.toValueString(v) + } + ).toMap) + } else { + "" + } - val post = new HttpPost(base + appended) - addHeaders(post, row) - getValueOpt(row, subscriptionRegion).foreach(post.setHeader("Ocp-Apim-Subscription-Region", _)) + val post = new HttpPost(base + appended) + addHeaders(post, row) + getValueOpt(row, subscriptionRegion).foreach(post.setHeader("Ocp-Apim-Subscription-Region", _)) - val json = texts.map(s => Map("Text" -> s)).toJson.compactPrint - post.setEntity(new StringEntity(json, "UTF-8")) - Some(post) + val json = version match { + case TranslatorApiVersion.V3 => texts.map(s => Map("Text" -> s)).toJson.compactPrint + case TranslatorApiVersion.V2026 => + val inputs = texts.map(s => JsObject("text" -> JsString(s))) + JsObject("inputs" -> JsArray(inputs.toVector)).compactPrint + } + post.setEntity(new StringEntity(json, "UTF-8")) + Some(post) + } } } } @@ -127,6 +145,33 @@ abstract class TextTranslatorBase(override val uid: String) extends CognitiveSer with HasInternalJsonOutputParser with HasSubscriptionRegion with HasSetLocation with HasSetLinkedServiceUsingLocation { + val apiVersion = new Param[String]( + this, + "apiVersion", + "Translator Text API version.", + isValid = TranslatorApiVersion.Supported) + + setDefault(apiVersion -> TranslatorApiVersion.V3) + + def getApiVersion: String = $(apiVersion) + + def setApiVersion(v: String): this.type = { + require(TranslatorApiVersion.Supported(v), + s"Unsupported Translator API version '$v'. Supported versions: $supportedApiVersions") + set(apiVersion, v) + } + + private def supportedApiVersions: String = TranslatorApiVersion.Supported.toSeq.sorted.mkString(", ") + + protected def supportsApiVersion(version: String): Boolean = version == TranslatorApiVersion.V3 + + protected def validateApiVersion(version: String): Unit = { + require(TranslatorApiVersion.Supported(version), + s"Unsupported Translator API version '$version'. Supported versions: $supportedApiVersions") + require(supportsApiVersion(version), + s"${getClass.getSimpleName} is not available in Translator API $version. Use API 3.0 for this operation.") + } + override private[ml] def internalServiceType: String = "texttranslation" protected def reshapeColumns(schema: StructType, parameterNames: Seq[String]) @@ -195,7 +240,10 @@ abstract class TextTranslatorBase(override val uid: String) extends CognitiveSer } override protected def getInternalTransformer(schema: StructType): PipelineModel = - customGetInternalTransformer(schema, Seq("text")) + { + validateApiVersion(getApiVersion) + customGetInternalTransformer(schema, Seq("text")) + } override def setLocation(v: String): this.type = { setSubscriptionRegion(v) @@ -215,51 +263,113 @@ class Translate(override val uid: String) extends TextTranslatorBase(uid) def urlPath: String = "translate" + override protected def supportsApiVersion(version: String): Boolean = + TranslatorApiVersion.Supported(version) + override protected def contentType: Row => String = { _ => "application/json; charset=UTF-8" } + private def v3Query(row: Row): String = { + val urlParams = getUrlParams + .asInstanceOf[Array[ServiceParam[Any]]] + if (urlParams.isEmpty) { + "" + } else { + "&" + URLEncodingUtils.format(urlParams.flatMap(p => + getValueOpt(row, p).map { + val pName = p.name match { + case "fromLanguage" => "from" + case "toLanguage" => "to" + case s => s + } + v => pName -> p.toValueString(v) + } + ).toMap) + } + } + + private def v2026Payload(row: Row, texts: Seq[String]): String = { + val v3OnlyValueRequested = getValueOpt(row, includeAlignment).contains(true) || + getValueOpt(row, includeSentenceLength).contains(true) || + getValueOpt(row, suggestedFrom).exists(_.trim.nonEmpty) + require(!v3OnlyValueRequested, + "includeAlignment, includeSentenceLength, and suggestedFrom are only supported by Translator API 3.0.") + val sourceLanguage = getValueOpt(row, fromLanguage).map(JsString(_)) + val sourceScript = getValueOpt(row, fromScript).map(JsString(_)) + val sourceTextType = getValueOpt(row, textType).map { value => + value match { + case "plain" => JsString("Plain") + case "html" => JsString("Html") + case _ => throw new IllegalArgumentException( + s"Invalid textType '$value'. Supported values are plain and html.") + } + } + val targetScript = getValueOpt(row, toScript).map(JsString(_)) + val deploymentName = getValueOpt(row, category).filterNot(_ == "general").map(JsString(_)) + val fallback = getValueOpt(row, allowFallback).filterNot(identity).map(JsBoolean(_)) + val action = getValueOpt(row, profanityAction).filterNot(_ == "NoAction").map(JsString(_)) + val marker = getValueOpt(row, profanityMarker).filterNot(_ == "Asterisk").map(JsString(_)) + val targetLanguages = getValue(row, toLanguage) + .flatMap(Option(_)) + .filter(_.trim.nonEmpty) + require(targetLanguages.nonEmpty, + "Translator API 2026-06-06 requires at least one non-blank target language.") + val targets = targetLanguages.map { language => + JsObject(Map("language" -> JsString(language)) ++ + targetScript.map("script" -> _) ++ + deploymentName.map("deploymentName" -> _) ++ + fallback.map("allowFallback" -> _) ++ + action.map("profanityAction" -> _) ++ + marker.map("profanityMarker" -> _)) + } + val inputs = texts.map { textValue => + JsObject(Map("text" -> JsString(textValue), "targets" -> JsArray(targets.toVector)) ++ + sourceLanguage.map("language" -> _) ++ + sourceScript.map("script" -> _) ++ + sourceTextType.filterNot(_ == JsString("Plain")).map("textType" -> _)) + } + JsObject("inputs" -> JsArray(inputs.toVector)).compactPrint + } + override protected def inputFunc(schema: StructType): Row => Option[HttpRequestBase] = { { row: Row => if (shouldSkip(row)) { None - } else if (getValue(row, text).forall(Option(_).isEmpty)) { - None - } else if (getValue(row, toLanguage).forall(Option(_).isEmpty)) { - None } else { - val urlParams: Array[ServiceParam[Any]] = - getUrlParams.asInstanceOf[Array[ServiceParam[Any]]] - + val version = getApiVersion + validateApiVersion(version) val texts = getValue(row, text) - - val base = getUrl + "?api-version=3.0" - val appended = if (!urlParams.isEmpty) { - "&" + URLEncodingUtils.format(urlParams.flatMap(p => - getValueOpt(row, p).map { - val pName = p.name match { - case "fromLanguage" => "from" - case "toLanguage" => "to" - case s => s - } - v => pName -> p.toValueString(v) - } - ).toMap) + .flatMap(Option(_)) + .filter(value => version == TranslatorApiVersion.V3 || value.trim.nonEmpty) + val targetsMissing = version == TranslatorApiVersion.V3 && + getValue(row, toLanguage).forall(Option(_).isEmpty) + if (texts.isEmpty || targetsMissing) { + None } else { - "" - } + val base = getUrl + s"?api-version=$version" + val appended = if (version == TranslatorApiVersion.V3) v3Query(row) else "" - val post = new HttpPost(base + appended) - addHeaders(post, row) - getValueOpt(row, subscriptionRegion).foreach(post.setHeader("Ocp-Apim-Subscription-Region", _)) + val post = new HttpPost(base + appended) + addHeaders(post, row) + getValueOpt(row, subscriptionRegion).foreach(post.setHeader("Ocp-Apim-Subscription-Region", _)) - val json = texts.map(s => Map("Text" -> s)).toJson.compactPrint - post.setEntity(new StringEntity(json, "UTF-8")) - Some(post) + val json = version match { + case TranslatorApiVersion.V3 => + texts.map(s => Map("Text" -> s)).toJson.compactPrint + case TranslatorApiVersion.V2026 => + v2026Payload(row, texts) + } + post.setEntity(new StringEntity(json, "UTF-8")) + Some(post) + } } } } override protected def getInternalTransformer(schema: StructType): PipelineModel = - customGetInternalTransformer(schema, Seq("text", "toLanguage")) + { + validateApiVersion(getApiVersion) + customGetInternalTransformer(schema, Seq("text", "toLanguage")) + } val toLanguage = new ServiceParam[Seq[String]](this, "toLanguage", "Specifies the language of the output text. The target language must be one of the supported languages" + @@ -377,7 +487,10 @@ class Translate(override val uid: String) extends TextTranslatorBase(uid) includeSentenceLength -> Left(false), allowFallback -> Left(true)) - override def responseDataType: DataType = ArrayType(TranslateResponse.schema) + override def responseDataType: DataType = getApiVersion match { + case TranslatorApiVersion.V3 => ArrayType(TranslateResponse.schema) + case TranslatorApiVersion.V2026 => TranslateResponseV2026.schema + } } object Transliterate extends ComplexParamsReadable[Transliterate] @@ -390,6 +503,9 @@ class Transliterate(override val uid: String) extends TextTranslatorBase(uid) def urlPath: String = "transliterate" + override protected def supportsApiVersion(version: String): Boolean = + TranslatorApiVersion.Supported(version) + val language = new ServiceParam[String](this, "language", "Language tag identifying the" + " language of the input text. If a code is not specified, automatic language detection will be applied.", isRequired = true, isURLParam = true) @@ -412,7 +528,10 @@ class Transliterate(override val uid: String) extends TextTranslatorBase(uid) def setToScriptCol(v: String): this.type = setVectorParam(toScript, v) - override def responseDataType: DataType = ArrayType(TransliterateResponse.schema) + override def responseDataType: DataType = getApiVersion match { + case TranslatorApiVersion.V3 => ArrayType(TransliterateResponse.schema) + case TranslatorApiVersion.V2026 => TransliterateResponseV2026.schema + } } object Detect extends ComplexParamsReadable[Detect] @@ -516,7 +635,9 @@ class DictionaryExamples(override val uid: String) extends TextTranslatorBase(ui None else { - val base = getUrl + "?api-version=3.0" + val version = getApiVersion + validateApiVersion(version) + val base = getUrl + s"?api-version=$version" val appended = if (!urlParams.isEmpty) { "&" + URLEncodingUtils.format(urlParams.flatMap(p => getValueOpt(row, p).map { @@ -553,7 +674,74 @@ class DictionaryExamples(override val uid: String) extends TextTranslatorBase(ui override protected def prepareEntity: Row => Option[AbstractHttpEntity] = { _ => None } override protected def getInternalTransformer(schema: StructType): PipelineModel = - customGetInternalTransformer(schema, Seq("textAndTranslation")) + { + validateApiVersion(getApiVersion) + customGetInternalTransformer(schema, Seq("textAndTranslation")) + } override def responseDataType: DataType = ArrayType(DictionaryExamplesResponse.schema) } + +object Languages extends ComplexParamsReadable[Languages] + +class Languages(override val uid: String) extends TextTranslatorBase(uid) + with HasCognitiveServiceInput with SynapseMLLogging { + logClass(FeatureNames.AiServices.Translate) + + def this() = this(Identifiable.randomUID("Languages")) + + override def urlPath: String = "languages" + + override protected def supportsApiVersion(version: String): Boolean = + TranslatorApiVersion.Supported(version) + + val scope = new StringArrayParam( + this, + "scope", + "Translator language groups to return. API 3.0 supports translation, transliteration, and dictionary; " + + "API 2026-06-06 supports translation, transliteration, and models.", + values => values.nonEmpty && + values.forall(Set("translation", "transliteration", "dictionary", "models"))) + + def getScope: Array[String] = $(scope) + + def setScope(v: Array[String]): this.type = set(scope, v) + + def setScope(v: Seq[String]): this.type = set(scope, v.toArray) + + def setScope(v: String): this.type = set(scope, Array(v)) + + private def validateScope(version: String, values: Seq[String]): Unit = { + val allowed = version match { + case TranslatorApiVersion.V3 => Set("translation", "transliteration", "dictionary") + case TranslatorApiVersion.V2026 => Set("translation", "transliteration", "models") + } + require(values.forall(allowed), + s"Translator API $version supports these language scopes: ${allowed.toSeq.sorted.mkString(", ")}") + } + + override protected def inputFunc(schema: StructType): Row => Option[HttpRequestBase] = { + { row: Row => + val version = getApiVersion + validateApiVersion(version) + val selectedScope = this.get(scope).map(_.toSeq) + selectedScope.foreach(validateScope(version, _)) + val query = Seq(Some("api-version" -> version), selectedScope.map("scope" -> _.mkString(","))).flatten + val request = new HttpGet(getUrl + "?" + URLEncodingUtils.format(query.toMap)) + addHeaders(request, row) + getValueOpt(row, subscriptionRegion).foreach(request.setHeader("Ocp-Apim-Subscription-Region", _)) + Some(request) + } + } + + override protected def prepareEntity: Row => Option[AbstractHttpEntity] = { _ => None } + + override protected def getInternalTransformer(schema: StructType): PipelineModel = { + val version = getApiVersion + validateApiVersion(version) + get(scope).foreach(values => validateScope(version, values)) + customGetInternalTransformer(schema, Seq.empty) + } + + override def responseDataType: DataType = TranslatorLanguagesResponse.schema +} diff --git a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/translate/TranslatorSchemas.scala b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/translate/TranslatorSchemas.scala index 7af6f4c6460..a582b881a3c 100644 --- a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/translate/TranslatorSchemas.scala +++ b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/translate/TranslatorSchemas.scala @@ -28,6 +28,22 @@ case class SentLen(srcSentLen: Seq[Int], transSentLen: Seq[Int]) case class SourceText(text: String) +object TranslateResponseV2026 extends SparkBindings[TranslateResponseV2026] + +case class TranslateResponseV2026(value: Seq[TranslateResultV2026]) + +case class TranslateResultV2026(detectedLanguage: Option[TranslatorDetectedLanguage], + translations: Seq[TranslationV2026]) + +case class TranslationV2026(language: String, + text: String, + script: Option[String], + sourceCharacters: Option[Long], + instructionTokens: Option[Long], + sourceTokens: Option[Long], + targetTokens: Option[Long], + responseTokens: Option[Long]) + object DetectResponse extends SparkBindings[DetectResponse] case class DetectResponse(language: String, @@ -49,6 +65,10 @@ object TransliterateResponse extends SparkBindings[TransliterateResponse] case class TransliterateResponse(text: String, script: String) +object TransliterateResponseV2026 extends SparkBindings[TransliterateResponseV2026] + +case class TransliterateResponseV2026(value: Seq[TransliterateResponse]) + object DictionaryLookupResponse extends SparkBindings[DictionaryLookupResponse] case class DictionaryLookupResponse(normalizedSource: String, @@ -78,6 +98,41 @@ case class DictionaryExamplesResponse(normalizedSource: String, case class Example(sourcePrefix: String, sourceTerm: String, sourceSuffix: String, targetPrefix: String, targetTerm: String, targetSuffix: String) +object TranslatorLanguagesResponse extends SparkBindings[TranslatorLanguagesResponse] + +case class TranslatorLanguagesResponse( + translation: Option[Map[String, TranslatorLanguage]], + transliteration: Option[Map[String, TranslatorTransliterationLanguage]], + dictionary: Option[Map[String, TranslatorDictionaryLanguage]], + models: Option[Seq[String]]) + +case class TranslatorLanguage(name: String, + nativeName: String, + dir: String, + models: Option[Seq[String]]) + +case class TranslatorTransliterationLanguage(name: String, + nativeName: String, + scripts: Seq[TranslatorSourceScript]) + +case class TranslatorSourceScript(code: String, + name: String, + nativeName: String, + dir: String, + toScripts: Seq[TranslatorTargetScript]) + +case class TranslatorTargetScript(code: String, name: String, nativeName: String, dir: String) + +case class TranslatorDictionaryLanguage(name: String, + nativeName: String, + dir: String, + translations: Seq[TranslatorTargetDictionaryLanguage]) + +case class TranslatorTargetDictionaryLanguage(name: String, + nativeName: String, + dir: String, + code: String) + case class DocumentTranslationInput(inputs: Seq[BatchRequest]) case class BatchRequest(source: SourceInput, diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/language/AnalyzeTextCoreOfflineSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/language/AnalyzeTextCoreOfflineSuite.scala index c54c64db460..6fb0f8dbc44 100644 --- a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/language/AnalyzeTextCoreOfflineSuite.scala +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/language/AnalyzeTextCoreOfflineSuite.scala @@ -69,6 +69,10 @@ class AnalyzeTextCoreOfflineSuite extends AnyFunSuite { } } + test("analyze text defaults to the latest GA API version") { + assert(new AnalyzeText().getApiVersion == "2024-11-01") + } + test("analyze text request-building is deterministic for language detection") { val transformer = new TestableAnalyzeText() .setKind("LanguageDetection") diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/translate/TextTranslatorCoreSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/translate/TextTranslatorCoreSuite.scala index 25abe5f1091..695b2efd2f2 100644 --- a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/translate/TextTranslatorCoreSuite.scala +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/translate/TextTranslatorCoreSuite.scala @@ -4,11 +4,12 @@ package com.microsoft.azure.synapse.ml.services.translate import com.microsoft.azure.synapse.ml.core.test.base.TestBase -import org.apache.http.client.methods.HttpPost +import org.apache.http.client.methods.{HttpGet, HttpPost, HttpRequestBase} import org.apache.http.util.EntityUtils import org.apache.spark.sql.Row import org.apache.spark.sql.types.ArrayType import org.apache.spark.sql.types.StructType +import spray.json._ import java.net.URLDecoder @@ -42,12 +43,17 @@ private[translate] class TestableDictionaryExamples extends DictionaryExamples { inputFunc(schema)(row).map(_.asInstanceOf[HttpPost]) } +private[translate] class TestableLanguages extends Languages { + def buildRequest(schema: StructType, row: Row): Option[HttpGet] = + inputFunc(schema)(row).map(_.asInstanceOf[HttpGet]) +} + class TextTranslatorCoreSuite extends TestBase { import spark.implicits._ - private def toQueryMap(post: HttpPost): Map[String, String] = { - Option(post.getURI.getRawQuery).toSeq.flatMap(_.split("&")).map { kv => + private def toQueryMap(request: HttpRequestBase): Map[String, String] = { + Option(request.getURI.getRawQuery).toSeq.flatMap(_.split("&")).map { kv => val pair = kv.split("=", 2) val key = URLDecoder.decode(pair(0), "UTF-8") val value = if (pair.length > 1) URLDecoder.decode(pair(1), "UTF-8") else "" @@ -69,6 +75,7 @@ class TextTranslatorCoreSuite extends TestBase { test("translate defaults are deterministic") { val t = new Translate() + assert(t.getApiVersion == "3.0") assert(t.getOrDefault(t.textType) == Left("plain")) assert(t.getOrDefault(t.category) == Left("general")) assert(t.getOrDefault(t.profanityAction) == Left("NoAction")) @@ -78,6 +85,14 @@ class TextTranslatorCoreSuite extends TestBase { assertResult(Left(true))(t.getOrDefault(t.allowFallback)) } + test("translator API version validation is deterministic") { + val error = intercept[IllegalArgumentException] { + new Translate().setApiVersion("2025-10-01-preview") + } + assert(error.getMessage.contains("Supported versions: 2026-06-06, 3.0")) + assert(!classOf[Translate].getMethods.exists(_.getName == "setApiVersionCol")) + } + test("translate rejects invalid enum parameters") { intercept[IllegalArgumentException] { new Translate().setTextType("markdown") @@ -135,6 +150,114 @@ class TextTranslatorCoreSuite extends TestBase { assert(t.buildRequest(nullToDf.schema, nullToDf.head()).isEmpty) } + test("translate builds the 2026 request body and response schema") { + val df = Seq((Seq("hello", "world"), Seq("de", "fr"), "en")) + .toDF("text", "toLanguage", "fromLanguage") + + val t = new TestableTranslate() + .setApiVersion("2026-06-06") + .setSubscriptionKey("fake-key") + .setLocation("eastus") + .setTextCol("text") + .setToLanguageCol("toLanguage") + .setFromLanguageCol("fromLanguage") + + val request = t.buildRequest(df.schema, df.head()).get + assert(toQueryMap(request) == Map("api-version" -> "2026-06-06")) + val body = EntityUtils.toString(request.getEntity, "UTF-8").parseJson.asJsObject + val inputs = body.fields("inputs").asInstanceOf[JsArray].elements + assert(inputs.map(_.asJsObject.fields("text")) == Seq(JsString("hello"), JsString("world"))) + inputs.foreach { input => + assert(input.asJsObject.fields("language") == JsString("en")) + assert(input.asJsObject.fields("targets").asInstanceOf[JsArray].elements == + Seq(JsObject("language" -> JsString("de")), JsObject("language" -> JsString("fr")))) + } + assert(t.responseDataType == TranslateResponseV2026.schema) + } + + test("translate filters invalid 2026 text and target array entries") { + val df = Seq((Seq("hello", null, " "), Seq("de", null, " "))) //scalastyle:ignore null + .toDF("text", "toLanguage") + val t = new TestableTranslate() + .setApiVersion("2026-06-06") + .setLocation("eastus") + .setTextCol("text") + .setToLanguageCol("toLanguage") + + val request = t.buildRequest(df.schema, df.head()).get + val inputs = EntityUtils.toString(request.getEntity, "UTF-8") + .parseJson.asJsObject.fields("inputs").asInstanceOf[JsArray].elements + assert(inputs.map(_.asJsObject.fields("text")) == Seq(JsString("hello"))) + assert(inputs.head.asJsObject.fields("targets").asInstanceOf[JsArray].elements == + Seq(JsObject("language" -> JsString("de")))) + + val blankTargets = Seq((Seq("hello"), Seq(null, " "))) //scalastyle:ignore null + .toDF("text", "toLanguage") + val error = intercept[IllegalArgumentException] { + t.buildRequest(blankTargets.schema, blankTargets.head()) + } + assert(error.getMessage.contains("at least one non-blank target language")) + } + + test("translate maps compatible 2026 controls and rejects removed controls") { + val request = new TestableTranslate() + .setApiVersion("2026-06-06") + .setLocation("eastus") + .setText("hello") + .setToLanguage("es") + .setFromLanguage("en") + .setFromScript("Latn") + .setToScript("Latn") + .setTextType("html") + .setCategory("custom-model") + .setAllowFallback(false) + .setProfanityAction("Marked") + .setProfanityMarker("Tag") + .buildRequest(StructType(Seq.empty), Row.empty) + .get + val body = EntityUtils.toString(request.getEntity, "UTF-8") + assert(body.contains(""""deploymentName":"custom-model"""")) + assert(body.contains(""""allowFallback":false""")) + assert(body.contains(""""profanityAction":"Marked"""")) + assert(body.contains(""""profanityMarker":"Tag"""")) + assert(body.contains(""""textType":"Html"""")) + + val neutralRequest = new TestableTranslate() + .setApiVersion("2026-06-06") + .setLocation("eastus") + .setText("hello") + .setToLanguage("es") + .setIncludeAlignment(false) + .setIncludeSentenceLength(false) + .setSuggestedFrom(" ") + .buildRequest(StructType(Seq.empty), Row.empty) + assert(neutralRequest.nonEmpty) + + val error = intercept[IllegalArgumentException] { + new TestableTranslate() + .setApiVersion("2026-06-06") + .setLocation("eastus") + .setText("hello") + .setToLanguage("es") + .setIncludeAlignment(true) + .buildRequest(StructType(Seq.empty), Row.empty) + } + assert(error.getMessage.contains("only supported by Translator API 3.0")) + + val invalidTextType = Seq((Seq("hello"), Seq("es"), "markdown")) + .toDF("text", "toLanguage", "textType") + val textTypeError = intercept[IllegalArgumentException] { + new TestableTranslate() + .setApiVersion("2026-06-06") + .setLocation("eastus") + .setTextCol("text") + .setToLanguageCol("toLanguage") + .setTextTypeCol("textType") + .buildRequest(invalidTextType.schema, invalidTextType.head()) + } + assert(textTypeError.getMessage.contains("Invalid textType 'markdown'")) + } + test("translate transformSchema adds output and error columns without temp columns") { val input = Seq(("hello", "de")).toDF("text", "toLanguage") val t = new Translate() @@ -180,6 +303,52 @@ class TextTranslatorCoreSuite extends TestBase { assert(EntityUtils.toString(request.getEntity, "UTF-8") == """[{"Text":"こんにちは"}]""") } + test("transliterate builds the wrapped 2026 request and response schema") { + val df = Seq((Seq("пример текста"), "ru", "Cyrl", "Latn")) + .toDF("text", "language", "fromScript", "toScript") + val t = new TestableTransliterate() + .setApiVersion("2026-06-06") + .setLocation("eastus") + .setTextCol("text") + .setLanguageCol("language") + .setFromScriptCol("fromScript") + .setToScriptCol("toScript") + + val request = t.buildRequest(df.schema, df.head()).get + val query = toQueryMap(request) + assert(query("api-version") == "2026-06-06") + assert(query("language") == "ru") + assert(query("fromScript") == "Cyrl") + assert(query("toScript") == "Latn") + assert(EntityUtils.toString(request.getEntity, "UTF-8") == + """{"inputs":[{"text":"пример текста"}]}""") + assert(t.responseDataType == TransliterateResponseV2026.schema) + } + + test("text-only request bodies ignore null entries") { + val df = Seq(Seq("hello", null)) //scalastyle:ignore null + .toDF("text") + + val v3Request = new TestableDetect() + .setLocation("eastus") + .setTextCol("text") + .buildRequest(df.schema, df.head()) + .get + assert(EntityUtils.toString(v3Request.getEntity, "UTF-8") == """[{"Text":"hello"}]""") + + val v2026Request = new TestableTransliterate() + .setApiVersion("2026-06-06") + .setLocation("eastus") + .setTextCol("text") + .setLanguage("en") + .setFromScript("Latn") + .setToScript("Latn") + .buildRequest(df.schema, df.head()) + .get + assert(EntityUtils.toString(v2026Request.getEntity, "UTF-8") == + """{"inputs":[{"text":"hello"}]}""") + } + test("detect and breaksentence request building is deterministic offline") { val detectDf = Seq(Seq("hello", "world")).toDF("text") val detectRequest = new TestableDetect() @@ -241,6 +410,67 @@ class TextTranslatorCoreSuite extends TestBase { assert(EntityUtils.toString(examplesRequest.getEntity, "UTF-8") == """[{"Text":"fly","Translation":"volar"}]""") } + test("v3-only operations reject Translator API 2026-06-06") { + val textOnly = Seq("hello").toDF("text") + val removed = Seq( + new Detect().setTextCol("text"), + new BreakSentence().setTextCol("text"), + new DictionaryLookup().setTextCol("text").setFromLanguage("en").setToLanguage("es")) + + removed.foreach { transformer => + transformer.setApiVersion("2026-06-06") + val error = intercept[IllegalArgumentException] { + transformer.transformSchema(textOnly.schema) + } + assert(error.getMessage.contains("is not available in Translator API 2026-06-06")) + } + + val examples = new DictionaryExamples() + .setApiVersion("2026-06-06") + .setTextAndTranslation(TextAndTranslation("fly", "volar")) + .setFromLanguage("en") + .setToLanguage("es") + val error = intercept[IllegalArgumentException] { + examples.transformSchema(StructType(Seq.empty)) + } + assert(error.getMessage.contains("is not available in Translator API 2026-06-06")) + } + + test("languages supports v3 and 2026 requests") { + val v3 = new TestableLanguages() + .setLocation("eastus") + .setScope(Seq("translation", "dictionary")) + .buildRequest(StructType(Seq.empty), Row.empty) + .get + assert(v3.getURI.getPath.endsWith("/languages")) + assert(toQueryMap(v3) == Map( + "api-version" -> "3.0", + "scope" -> "translation,dictionary")) + + val latest = new TestableLanguages() + .setApiVersion("2026-06-06") + .setLocation("eastus") + .setScope("models") + .buildRequest(StructType(Seq.empty), Row.empty) + .get + assert(toQueryMap(latest) == Map("api-version" -> "2026-06-06", "scope" -> "models")) + assert(latest.getFirstHeader("Ocp-Apim-Subscription-Region").getValue == "eastus") + assert(new Languages().responseDataType == TranslatorLanguagesResponse.schema) + assert(TranslatorLanguagesResponse.schema("models").dataType == ArrayType(org.apache.spark.sql.types.StringType)) + + val v3Error = intercept[IllegalArgumentException] { + new Languages().setScope("models").transformSchema(StructType(Seq.empty)) + } + assert(v3Error.getMessage.contains("dictionary, translation, transliteration")) + val latestError = intercept[IllegalArgumentException] { + new Languages() + .setApiVersion("2026-06-06") + .setScope("dictionary") + .transformSchema(StructType(Seq.empty)) + } + assert(latestError.getMessage.contains("models, translation, transliteration")) + } + test("dictionary examples request building supports scalar text and translation input") { val request = new TestableDictionaryExamples() .setLocation("eastus") diff --git a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/translate/TranslatorSuite.scala b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/translate/TranslatorSuite.scala index 6a0bccc74b7..c7c7b99c588 100644 --- a/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/translate/TranslatorSuite.scala +++ b/cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/translate/TranslatorSuite.scala @@ -409,6 +409,21 @@ class DictionaryExamplesSuite extends TransformerFuzzing[DictionaryExamples] override def reader: MLReadable[_] = DictionaryExamples } +class LanguagesSuite extends TransformerFuzzing[Languages] + with TranslatorUtils { + override val compareDataInSerializationTest: Boolean = false + + def languages: Languages = new Languages() + .setLocation("eastus") + .setScope("translation") + .setOutputCol("languages") + + override def testObjects(): Seq[TestObject[Languages]] = + Seq(new TestObject(languages, emptyDf)) + + override def reader: MLReadable[_] = Languages +} + // TODO add this test back in when fixed //class DocumentTranslatorSuite extends TransformerFuzzing[DocumentTranslator] // with TranslatorKey with Flaky { diff --git a/docs/Quick Examples/transformers/cognitive/_Translator.md b/docs/Quick Examples/transformers/cognitive/_Translator.md index 648d0dad0d3..c6e6141dea9 100644 --- a/docs/Quick Examples/transformers/cognitive/_Translator.md +++ b/docs/Quick Examples/transformers/cognitive/_Translator.md @@ -5,6 +5,14 @@ import DocTable from "@theme/DocumentationTable"; ## Translator +Translator API `3.0` remains the default for backward compatibility. `Translate`, +`Transliterate`, and `Languages` can opt into API `2026-06-06` with +`.setApiVersion("2026-06-06")`. The newer API uses an `inputs` request object +and wraps Translate and Transliterate results in a `value` field, so update +downstream column expressions accordingly. `Detect`, `BreakSentence`, +`DictionaryLookup`, and `DictionaryExamples` are not available in API +`2026-06-06` and must continue using API `3.0`. + ### Translate