What is the most appropriate strategy for deploying Azure Functions app with a customer Docker container?

  Kiến thức lập trình

I’m working on a C# .NET Azure Functions app.
This app follows a common pattern: Http Starter, Durable Orchestrator, Activity Function. Activity function requires access to “Simba Spark ODBC Driver” in order to retrieve data from Azure Databricks Sql Warehouse. Because Azure Functions are “serverless”, this Activity function needs to be deployed to run in a custom Docker container, which will have this ODBC driver installed and configured. I have two questions related to packaging and deployment of this app:

  1. Given that only one out of three functions in this app requires a customer container, should I deploy this function as a separate functions app and have the other two functions in a separate app? If I did this, how will the Durable Orchestrator be able to invoke the Activity function which belongs to a different functions app?

  2. If I was to build a single custom Docker container to host all three functions within a single app – how would the horizontal scaling be accomplished since all three functions would have to be replicated instead of just the Activity function?

Another words – what is the most appropriate packaging and deployment strategy for this use case?

Here are the functions:

OrchestratorStarter.cs:

using Microsoft.DurableTask.Client;
using Microsoft.Extensions.Logging;

namespace DurableOrchestratorDotnet8Linux
{
    public static class OrchestratorStarter
    {
        [Function("Orchestrator_HttpStart")]
        public static async Task<HttpResponseData> HttpStart(
            [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req,
            [DurableClient] DurableTaskClient client,
            FunctionContext executionContext)
        {
            ILogger logger = executionContext.GetLogger("Orchestrator_HttpStart");

            // Function input comes from the request content.
            string instanceId = await client.ScheduleNewOrchestrationInstanceAsync(
                nameof(Orchestrator));

            logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId);

            // Returns an HTTP 202 response with an instance management payload.
            // See https://learn.microsoft.com/azure/azure-functions/durable/durable-functions-http-api#start-orchestration
            return await client.CreateCheckStatusResponseAsync(req, instanceId);
        }
    }
}

Orchestrator.cs:

using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Http;
using Microsoft.DurableTask;
using Microsoft.DurableTask.Client;
using Microsoft.Extensions.Logging;
using System.Data;
using System.Data.Odbc;
using System.Runtime.InteropServices;
using System.IO;

namespace DurableOrchestratorDotnet8Linux
{
    public static class Orchestrator
    {
        [Function(nameof(Orchestrator))]
        public static async Task<List<string>> RunOrchestrator(
            [OrchestrationTrigger] TaskOrchestrationContext context)
        {
            ILogger logger = context.CreateReplaySafeLogger(nameof(Orchestrator));
            logger.LogInformation("Invoking activity function.");
            var outputs = new List<string>();
            outputs.Add(await context.CallActivityAsync<string>(nameof(ActivityFunction), "Databricks"));
        
            return outputs;
        }

        
    }
}

Activity.cs (the one which needs the customer container to gain access to the ODBC driver):

using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Http;
using Microsoft.DurableTask;
using Microsoft.DurableTask.Client;
using Microsoft.Extensions.Logging;
using System.Data.Odbc;

namespace DurableOrchestratorDotnet8Linux
{
    public static class ActivityFunction
    {
        [Function(nameof(ActivityFunction))]
        public static string RetrieveDatabricksData([ActivityTrigger] string name, FunctionContext executionContext)
        {
            ILogger logger = executionContext.GetLogger("SayHello");
            logger.LogInformation("DB Warehouse Saying hello to {name}.", name);

            var connectionString = "DSN=DatabricksWarehouse";
            var query = "SELECT c.customer_id, c.customer_name, c.customer_email, o.order_id, o.product, o.amount" +
                     " FROM Customers c" +
                     " JOIN Orders o ON c.customer_id = o.customer_id";

            List<string> records = new List<string>();

            try {
                    using (OdbcConnection connection = new OdbcConnection(connectionString))
                    {
                        try
                        {
                            connection.Open();

                            using (OdbcCommand command = new OdbcCommand(query, connection))
                            {
                                using (OdbcDataReader reader = command.ExecuteReader())
                                {
                                    while (reader.Read())
                                    {
                                        string record = reader.GetString(0) + " " + reader.GetString(1) +
                                                        " " + reader.GetString(2) + " " + reader.GetString(3) +
                                                        " " + reader.GetString(4) + " " + reader.GetString(5);
                                        records.Add(record);
                                    }
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            logger.LogError($"An error occurred: {ex.Message}");
                            name = ex.Message;
                        }
                    }
            }
            catch (Exception ex)
            {
                logger.LogError($"An error occurred: {ex.Message}");
                name = ex.Message;
            }
            return $"Hello {name}!";
        }
    }
}

Dockerfile (with the ODBC driver installed):

# Use the official Microsoft Azure Functions base image
FROM mcr.microsoft.com/azure-functions/dotnet:4.0-appservice

# Install dependencies
RUN apt-get update && 
    apt-get install -y curl apt-transport-https unixodbc-dev libsasl2-modules-gssapi-mit && 
    curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && 
    curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list > /etc/apt/sources.list.d/mssql-release.list && 
    apt-get update && 
    ACCEPT_EULA=Y apt-get install -y msodbcsql17 && 
    apt-get clean && 
    rm -rf /var/lib/apt/lists/*
 
# Copy the local Simba Spark ODBC Driver .deb file into the image
COPY simbaspark_2.8.0.1002-2_amd64.deb /tmp/simbaspark_2.8.0.1002-2_amd64.deb
COPY ./ini/odbc/* /usr/local/odbc/
COPY ./ini/simba.sparkodbc.ini /etc/
 
# Install dependencies required by Simba Spark ODBC Driver
RUN apt-get update && 
    apt-get install -y libsasl2-modules-gssapi-mit
 
# Install the Simba Spark ODBC Driver
RUN dpkg -i /tmp/simbaspark_2.8.0.1002-2_amd64.deb && 
    apt-get install -f -y && 
    rm /tmp/simbaspark_2.8.0.1002-2_amd64.deb
 
# Set environment variables for ODBC
ENV ODBCINI=/usr/local/odbc/odbc.ini
ENV ODBCSYSINI=/usr/local/odbc/
ENV SPARKINI=/etc/simba.sparkodbc.ini
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
 
# Copy the function app files to the image
COPY . /home/site/wwwroot
 
# Expose the port Azure Functions runtime listens to
EXPOSE 80

LEAVE A COMMENT