I now realize the fault of my ways, was using the certificates from the unit tests and those refer to a specific endpoint, also had the wrong root certificate.
I’ve created a self signed certificate and keys and everything seems to be working. Will try to add this as an example to the grpc repo so that others can benefit.

greeter client.cc

std::string ca_cert = R"(-----BEGIN CERTIFICATE-----
MIIC/zCCAeegAwIBAgIUEgsv2XdOGiDu0s8OmDVT6vpvjk4wDQYJKoZIhvcNAQEL
BQAwDzENMAsGA1UEAwwETXlDQTAeFw0yNDA5MDkxNTE4MjJaFw0yNTA5MDkxNTE4
MjJaMA8xDTALBgNVBAMMBE15Q0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQC0hoNdKMfmZkp3RouARUsxED0Bu4OaOjZftvvoVdqMoMqJsJEyi4zAR+ho
0bs/0yGw4MQfQyXVtv/+P/0r+pSPFQeNweyPqzFKe0eMaBL2DNxWHNbAUCvivUI7
2oGOul1M7rupI2bEKRUEnq8oAjV8f1HVwWY6huBH4FLTdsggsnc1WWqjZceNDGVz
Qq3lo27apMAhfeLlVqgGXV1Z23u3vQ5/8xYnT/Gk/bvRfEr2LrFx7WFsBk7ajgQy
QXbmkHh0GVDbCAZCuJIk3YzbqTDiUwXdcFaDFcjsHoCojONq+IroR/sFs7/LHtjr
/IwkyO3sacXpMPX5I+mQPBXeTZihAgMBAAGjUzBRMB0GA1UdDgQWBBQzOqHNiqo2
euaxWILuUIj52NW2ujAfBgNVHSMEGDAWgBQzOqHNiqo2euaxWILuUIj52NW2ujAP
BgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAsGVCbMe7ClQCtYZFE
lakrC3rv8ue7ip6zNdXk2cyCTK5aEEqt/dP2EINyB4RWbXZLfpHC/oU4qW82jfrf
QCFqeIQj94gFTR2JDVMbYrYyJiUVsdcWnUvLWA3CLdRFquG96tmfPrAzVdhV0Utj
g0phovVcc5pyqeDwS1CqM5RncEFS+fHfvqvaf3EoSbKTdbngxLcXUYQEywo+KVlg
C8j5bTwasraqPXZjl5EAYXBhXx6J0t33UiJrS0XamJEOPOTGcAFaGKb95pbHBCX1
BeXo1uFx7Vtk8Xo56QhjkkOXFBJePZqzuK649icHWN+QKPOMO8469LNjWksn/Xrz
gHnL
-----END CERTIFICATE-----
)";

std::string server_cert = R"(-----BEGIN CERTIFICATE-----
MIICqjCCAZICFBXSuBVnXcJLFDq8kx64p4WjETeKMA0GCSqGSIb3DQEBCwUAMA8x
DTALBgNVBAMMBE15Q0EwHhcNMjQwOTA5MTUxOTE4WhcNMjUwOTA5MTUxOTE4WjAU
MRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQC6TtLbYnCuQa5SKOreDamYzE/Ka5mVoOFoh0T+hlGMIjjw+aB+TfnCa/3u
xbhkH7bXPBGmFTm92lPOMSOBJxaOx5cEaRdYoPTTZchBT8BRndB6dymj3U7mpwwS
WwOieHalv/WXm5TzodI+utTVIkmcvgL5WQuAQvVVSsqgwfxMf3F4CKutocsYlDhG
HcxQY/pUr4rGU21bhU9VH+h0uujuJmnLPKkILvbRX3cTcu3psLNvrcjdVBzHLEq4
gcMwxNAU7HQM1/7t/0s0w86g1D6aWYdPxwJ4R9cnWZ2e/u3J4e2mpSfDtkRtQ/PY
eW0F+bHGEl/UG2IHOud0EJFTQoG7AgMBAAEwDQYJKoZIhvcNAQELBQADggEBAGHd
R4eWG51mW/DAOa7m5+IIVgS2NhAXfsOotfwQeX2dY49Hkgaf63q1f1lfHc1HUjkR
Vg+8ypxUohtqzEM1RYDAHGFoiDFEdCNFvGqiCusS7Ql1OwAkECH3oU8P/2u+wgju
hdpnTiQvvUKObmTIJVYzzpuNmHWx758lD2X69aXH5bV2eTnhPlxsK+ct5YfgtUSJ
guJkvtbtxzeuoiUL3frYztyZN0xUw1j7FeTxyLJxYVPqluGUqkRKyLqcg8v5JTt8
YxatRU+lvptxLAOvBeLyCsr6fXwsMPH5UMNeH5mBEIg9Vec1XiyS8XmX/eTE2n+p
shiBdkDyfSTUBKPPjcc=
-----END CERTIFICATE-----
)";

std::string server_key = R"(-----BEGIN PRIVATE KEY-----
MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC6TtLbYnCuQa5S
KOreDamYzE/Ka5mVoOFoh0T+hlGMIjjw+aB+TfnCa/3uxbhkH7bXPBGmFTm92lPO
MSOBJxaOx5cEaRdYoPTTZchBT8BRndB6dymj3U7mpwwSWwOieHalv/WXm5TzodI+
utTVIkmcvgL5WQuAQvVVSsqgwfxMf3F4CKutocsYlDhGHcxQY/pUr4rGU21bhU9V
H+h0uujuJmnLPKkILvbRX3cTcu3psLNvrcjdVBzHLEq4gcMwxNAU7HQM1/7t/0s0
w86g1D6aWYdPxwJ4R9cnWZ2e/u3J4e2mpSfDtkRtQ/PYeW0F+bHGEl/UG2IHOud0
EJFTQoG7AgMBAAECggEABhLOajsfTM70qg84v/jWfhYD4hlppv28hhJ7tNzeweed
ec9bOdER0RKBgZMqRlXAjs88RdRvwI5wmkpkPXdhL/yZsCgCNtOobqSsHMa4yWIe
w/SaEgQgBQI4ovepCOUVUfnxWd2qwy9a2QUbgxVqnj8/78rFG+DaCFnFmUHH3nEn
MXXfixKY6oE11DErZXVGNGxwy6ygfMn5/nBI9EjkAtl649rqpqeGL6xP5vQwqumr
ZjLrrxjv+9yIBiW0FHZrW7i4dPF6tYwtA+X80KeJGPheIfUlw3dxD+FFJqhKRVuS
Mqh15aXJEliqzVu3Ap54EnJkALIAvpbSRpABEcv5IQKBgQD7/q4UqJ3FgAkF7I3n
N1gro4oQzqFdIM1LHTjAsM4N746gpNmNvFYD68qOLYSiWiQ7aOrEA2XBb+jDULRU
UDUqZ3vJDaVvrQ/baWSLYxUoJ8C0Iu6yA++DCsmKM38Jir4Pp5MkO5EWD3gN54iD
pRpb1YHL8imsh9gO16BsfrqLJQKBgQC9ROAx07XzTbsj2yWKFqz/20YwaF+183ys
/TVaqamBPJBIATYSUiMjxnKBFZVnTtRMzG9lG9ckEoCS2qCyoAkJztJsiqe138VS
68O5T91XChfyeibmWU53V3NM/3vyKXUVAjaj85mXei+4sosYjK8wjQ3qDaPIfgAP
THDu/GazXwKBgHUBdhcFi+xOXOIxSlpXqkro7oyLRQWW23vLH7To42Q5HUKeCJ31
GwNLEowdun4f2L71IjzNTwwYSD2YVYLokycTUbiy62QFOV2pfBP0d7hjbOi3Z5mk
liuEcLwI2S23DDT8nCewuNdDa30ZSpvFp42If3IRCSShFsMdf9GgrkE5AoGAAg/g
CWrvDomIQmm+zPRWSitPZnOcp1TRxOi1ThmPGTNZtw8cUbLHYzpkQPfFOuzm7zdC
920IOQJimDb9jTSlJJA2Rqx0C002zyJ2bWxrUulvPVsLVXMfobk4LlySMx80gVgW
1E5xG+9e2bpIPao6tmKzBhvD7wlAYupISLJDRC0CgYBgzyDOjf23s7XFnw51V72k
yzI5DW7YMrVoujQyk+S00wTmlEa2NARAJj7rPDZpGPW6VM0/n18fnT2raF+LHxeZ
O1+smifQPAzrRbj7R/hwdZuLyyQPiRqGZHaOfnBBaCxvklnBUsupujH0vTy0N1TO
gQpkG/wCNnIGHr6jcq3rKg==
-----END PRIVATE KEY-----
)";

    int main(int argc, char** argv) {
      absl::ParseCommandLine(argc, argv);
      // Instantiate the client. It requires a channel, out of which the actual RPCs
      // are created. This channel models a connection to an endpoint specified by
      // the argument "--target=" which is the only expected argument.
      std::string target_str = absl::GetFlag(FLAGS_target);
    
      const auto channel_credentials = CreateTlsChannelCredentials(ca_cert, server_cert, server_key); 
      GreeterClient greeter(grpc::CreateChannel(target_str, channel_credentials));
      std::string user("world");
      std::string reply = greeter.SayHello(user);
      std::cout << "Greeter received: " << reply << std::endl;
    
      return 0;
    }

greeter_server.cc

std::string root_cert = R"(-----BEGIN CERTIFICATE-----
MIIC/zCCAeegAwIBAgIUEgsv2XdOGiDu0s8OmDVT6vpvjk4wDQYJKoZIhvcNAQEL
BQAwDzENMAsGA1UEAwwETXlDQTAeFw0yNDA5MDkxNTE4MjJaFw0yNTA5MDkxNTE4
MjJaMA8xDTALBgNVBAMMBE15Q0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQC0hoNdKMfmZkp3RouARUsxED0Bu4OaOjZftvvoVdqMoMqJsJEyi4zAR+ho
0bs/0yGw4MQfQyXVtv/+P/0r+pSPFQeNweyPqzFKe0eMaBL2DNxWHNbAUCvivUI7
2oGOul1M7rupI2bEKRUEnq8oAjV8f1HVwWY6huBH4FLTdsggsnc1WWqjZceNDGVz
Qq3lo27apMAhfeLlVqgGXV1Z23u3vQ5/8xYnT/Gk/bvRfEr2LrFx7WFsBk7ajgQy
QXbmkHh0GVDbCAZCuJIk3YzbqTDiUwXdcFaDFcjsHoCojONq+IroR/sFs7/LHtjr
/IwkyO3sacXpMPX5I+mQPBXeTZihAgMBAAGjUzBRMB0GA1UdDgQWBBQzOqHNiqo2
euaxWILuUIj52NW2ujAfBgNVHSMEGDAWgBQzOqHNiqo2euaxWILuUIj52NW2ujAP
BgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAsGVCbMe7ClQCtYZFE
lakrC3rv8ue7ip6zNdXk2cyCTK5aEEqt/dP2EINyB4RWbXZLfpHC/oU4qW82jfrf
QCFqeIQj94gFTR2JDVMbYrYyJiUVsdcWnUvLWA3CLdRFquG96tmfPrAzVdhV0Utj
g0phovVcc5pyqeDwS1CqM5RncEFS+fHfvqvaf3EoSbKTdbngxLcXUYQEywo+KVlg
C8j5bTwasraqPXZjl5EAYXBhXx6J0t33UiJrS0XamJEOPOTGcAFaGKb95pbHBCX1
BeXo1uFx7Vtk8Xo56QhjkkOXFBJePZqzuK649icHWN+QKPOMO8469LNjWksn/Xrz
gHnL
-----END CERTIFICATE-----
)";

std::string server_cert = R"(-----BEGIN CERTIFICATE-----
MIICqjCCAZICFBXSuBVnXcJLFDq8kx64p4WjETeKMA0GCSqGSIb3DQEBCwUAMA8x
DTALBgNVBAMMBE15Q0EwHhcNMjQwOTA5MTUxOTE4WhcNMjUwOTA5MTUxOTE4WjAU
MRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQC6TtLbYnCuQa5SKOreDamYzE/Ka5mVoOFoh0T+hlGMIjjw+aB+TfnCa/3u
xbhkH7bXPBGmFTm92lPOMSOBJxaOx5cEaRdYoPTTZchBT8BRndB6dymj3U7mpwwS
WwOieHalv/WXm5TzodI+utTVIkmcvgL5WQuAQvVVSsqgwfxMf3F4CKutocsYlDhG
HcxQY/pUr4rGU21bhU9VH+h0uujuJmnLPKkILvbRX3cTcu3psLNvrcjdVBzHLEq4
gcMwxNAU7HQM1/7t/0s0w86g1D6aWYdPxwJ4R9cnWZ2e/u3J4e2mpSfDtkRtQ/PY
eW0F+bHGEl/UG2IHOud0EJFTQoG7AgMBAAEwDQYJKoZIhvcNAQELBQADggEBAGHd
R4eWG51mW/DAOa7m5+IIVgS2NhAXfsOotfwQeX2dY49Hkgaf63q1f1lfHc1HUjkR
Vg+8ypxUohtqzEM1RYDAHGFoiDFEdCNFvGqiCusS7Ql1OwAkECH3oU8P/2u+wgju
hdpnTiQvvUKObmTIJVYzzpuNmHWx758lD2X69aXH5bV2eTnhPlxsK+ct5YfgtUSJ
guJkvtbtxzeuoiUL3frYztyZN0xUw1j7FeTxyLJxYVPqluGUqkRKyLqcg8v5JTt8
YxatRU+lvptxLAOvBeLyCsr6fXwsMPH5UMNeH5mBEIg9Vec1XiyS8XmX/eTE2n+p
shiBdkDyfSTUBKPPjcc=
-----END CERTIFICATE-----
)";

std::string server_key = R"(-----BEGIN PRIVATE KEY-----
MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC6TtLbYnCuQa5S
KOreDamYzE/Ka5mVoOFoh0T+hlGMIjjw+aB+TfnCa/3uxbhkH7bXPBGmFTm92lPO
MSOBJxaOx5cEaRdYoPTTZchBT8BRndB6dymj3U7mpwwSWwOieHalv/WXm5TzodI+
utTVIkmcvgL5WQuAQvVVSsqgwfxMf3F4CKutocsYlDhGHcxQY/pUr4rGU21bhU9V
H+h0uujuJmnLPKkILvbRX3cTcu3psLNvrcjdVBzHLEq4gcMwxNAU7HQM1/7t/0s0
w86g1D6aWYdPxwJ4R9cnWZ2e/u3J4e2mpSfDtkRtQ/PYeW0F+bHGEl/UG2IHOud0
EJFTQoG7AgMBAAECggEABhLOajsfTM70qg84v/jWfhYD4hlppv28hhJ7tNzeweed
ec9bOdER0RKBgZMqRlXAjs88RdRvwI5wmkpkPXdhL/yZsCgCNtOobqSsHMa4yWIe
w/SaEgQgBQI4ovepCOUVUfnxWd2qwy9a2QUbgxVqnj8/78rFG+DaCFnFmUHH3nEn
MXXfixKY6oE11DErZXVGNGxwy6ygfMn5/nBI9EjkAtl649rqpqeGL6xP5vQwqumr
ZjLrrxjv+9yIBiW0FHZrW7i4dPF6tYwtA+X80KeJGPheIfUlw3dxD+FFJqhKRVuS
Mqh15aXJEliqzVu3Ap54EnJkALIAvpbSRpABEcv5IQKBgQD7/q4UqJ3FgAkF7I3n
N1gro4oQzqFdIM1LHTjAsM4N746gpNmNvFYD68qOLYSiWiQ7aOrEA2XBb+jDULRU
UDUqZ3vJDaVvrQ/baWSLYxUoJ8C0Iu6yA++DCsmKM38Jir4Pp5MkO5EWD3gN54iD
pRpb1YHL8imsh9gO16BsfrqLJQKBgQC9ROAx07XzTbsj2yWKFqz/20YwaF+183ys
/TVaqamBPJBIATYSUiMjxnKBFZVnTtRMzG9lG9ckEoCS2qCyoAkJztJsiqe138VS
68O5T91XChfyeibmWU53V3NM/3vyKXUVAjaj85mXei+4sosYjK8wjQ3qDaPIfgAP
THDu/GazXwKBgHUBdhcFi+xOXOIxSlpXqkro7oyLRQWW23vLH7To42Q5HUKeCJ31
GwNLEowdun4f2L71IjzNTwwYSD2YVYLokycTUbiy62QFOV2pfBP0d7hjbOi3Z5mk
liuEcLwI2S23DDT8nCewuNdDa30ZSpvFp42If3IRCSShFsMdf9GgrkE5AoGAAg/g
CWrvDomIQmm+zPRWSitPZnOcp1TRxOi1ThmPGTNZtw8cUbLHYzpkQPfFOuzm7zdC
920IOQJimDb9jTSlJJA2Rqx0C002zyJ2bWxrUulvPVsLVXMfobk4LlySMx80gVgW
1E5xG+9e2bpIPao6tmKzBhvD7wlAYupISLJDRC0CgYBgzyDOjf23s7XFnw51V72k
yzI5DW7YMrVoujQyk+S00wTmlEa2NARAJj7rPDZpGPW6VM0/n18fnT2raF+LHxeZ
O1+smifQPAzrRbj7R/hwdZuLyyQPiRqGZHaOfnBBaCxvklnBUsupujH0vTy0N1TO
gQpkG/wCNnIGHr6jcq3rKg==
-----END PRIVATE KEY-----
)";

void RunServer(uint16_t port) {
  std::string server_address = absl::StrFormat("0.0.0.0:%d", port);
  GreeterServiceImpl service;

  grpc::EnableDefaultHealthCheckService(true);
  grpc::reflection::InitProtoReflectionServerBuilderPlugin();
  ServerBuilder builder;

  // Listen on the given address without any authentication mechanism.
  const auto server_credentials = CreateTlsServerCredentials(root_cert, server_cert, server_key);
  builder.AddListeningPort(server_address, server_credentials); 
  // Register "service" as the instance through which we'll communicate with
  // clients. In this case it corresponds to an *synchronous* service.
  builder.RegisterService(&service);
  // Finally assemble the server.
  std::unique_ptr<Server> server(builder.BuildAndStart());
  std::cout << "Server listening on " << server_address << std::endl;

  // Wait for the server to shutdown. Note that some other thread must be
  // responsible for shutting down the server for this call to ever return.
  server->Wait();
}

certificate_verifier.h

#pragma once

#include <memory>

#include <grpcpp/grpcpp.h>

using grpc::experimental::ExternalCertificateVerifier;
using grpc::experimental::TlsCustomVerificationCheckRequest;

std::shared_ptr<grpc::ChannelCredentials> CreateTlsChannelCredentials(
  std::string ca_cert,
  std::string server_cert,
  std::string server_key);

std::shared_ptr<grpc::ServerCredentials> CreateTlsServerCredentials(
  std::string root_cert,
  std::string server_cert,
  std::string server_key);

class TestCertificateVerifier
    : public ExternalCertificateVerifier {
  public:
    explicit TestCertificateVerifier(bool success) : success_(success) {}
    ~TestCertificateVerifier() override {}

    bool Verify(TlsCustomVerificationCheckRequest* request,
      std::function<void(grpc::Status)> callback,
      grpc::Status* sync_status) override;
    void Cancel(TlsCustomVerificationCheckRequest* request) override {
    }
  private:
    bool success_ = false;
};

certificate_verifier.cc

#include "ab_certificate_verifier.h"

#include <iostream>
#include <string>

#include <grpcpp/security/credentials.h>
#include <grpcpp/security/tls_certificate_provider.h>

using grpc::experimental::TlsServerCredentials;
using grpc::experimental::TlsServerCredentialsOptions;
using grpc::experimental::StaticDataCertificateProvider;
using grpc::experimental::IdentityKeyCertPair;
using grpc::experimental::TlsCustomVerificationCheckRequest;

std::shared_ptr<grpc::ChannelCredentials> CreateTlsChannelCredentials(
    std::string ca_cert,
    std::string server_cert,
    std::string server_key) {
  std::vector<IdentityKeyCertPair> identity_pair;
  identity_pair.emplace_back(IdentityKeyCertPair{server_key, server_cert});

  auto certificate_provider = std::make_shared<StaticDataCertificateProvider>(ca_cert, identity_pair);
  grpc::experimental::TlsChannelCredentialsOptions options;
  options.set_certificate_provider(std::move(certificate_provider));
  options.watch_root_certs();
  options.watch_identity_key_cert_pairs();
  auto verifier = ExternalCertificateVerifier::Create<TestCertificateVerifier>(true);
  options.set_certificate_verifier(std::move(verifier));
  options.set_verify_server_certs(true);
  options.set_check_call_host(false);
  return grpc::experimental::TlsCredentials(options);
}

std::shared_ptr<grpc::ServerCredentials> CreateTlsServerCredentials(
    std::string root_cert,
    std::string server_cert,
    std::string server_key) {
  std::vector<IdentityKeyCertPair> identity_key_cert_pairs;
  identity_key_cert_pairs.emplace_back(IdentityKeyCertPair{server_key, server_cert});
  auto certificate_provider = std::make_shared<StaticDataCertificateProvider>(root_cert, identity_key_cert_pairs);
  TlsServerCredentialsOptions options(certificate_provider);
  options.set_certificate_provider(std::move(certificate_provider));

  auto verifier = ExternalCertificateVerifier::Create<TestCertificateVerifier>(true);
  options.set_certificate_verifier(std::move(verifier));
  
  options.watch_root_certs();
  options.set_root_cert_name("root");
  options.watch_identity_key_cert_pairs();
  options.set_identity_cert_name("identity");
  options.set_cert_request_type(
      GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY);
  return TlsServerCredentials(options);
}

bool TestCertificateVerifier::Verify(TlsCustomVerificationCheckRequest* request,
  std::function<void(grpc::Status)> callback,
  grpc::Status* sync_status) {
    if (!success_) {
      *sync_status = grpc::Status(grpc::StatusCode::UNAUTHENTICATED,
                                  "SyncCertificateVerifier failed");
    } else {
      *sync_status = grpc::Status(grpc::StatusCode::OK, "");
    }

    // Add custom verification here
    if(request->peer_cert().empty()) {
      std::cout << "Empty certificate" << std::endl;
      return false;
    }

    return true;
};

BUILD

licenses(["notice"])

cc_binary(
    name = "greeter_client",
    srcs = ["greeter_client.cc"],
    defines = ["BAZEL_BUILD"],
    deps = [
        "//:grpc++",
        "//examples/protos:helloworld_cc_grpc",
        "//examples/cpp/helloworld:ab_certificate_verifier",
        "@com_google_absl//absl/flags:flag",
        "@com_google_absl//absl/flags:parse",
    ],
)

cc_binary(
    name = "greeter_server",
    srcs = ["greeter_server.cc"],
    defines = ["BAZEL_BUILD"],
    deps = [
        "//:grpc++",
        "//:grpc++_reflection",
        "//examples/protos:helloworld_cc_grpc",
        "//examples/cpp/helloworld:ab_certificate_verifier",
        "@com_google_absl//absl/flags:flag",
        "@com_google_absl//absl/flags:parse",
        "@com_google_absl//absl/strings:str_format",
    ],
)

cc_library(
    name = "ab_certificate_verifier",
    srcs = ["ab_certificate_verifier.cc"],
    hdrs = ["ab_certificate_verifier.h"],
    defines = ["BAZEL_BUILD"],
    deps = [
        "//:grpc++",
        "//examples/protos:helloworld_cc_grpc",
        "@com_google_absl//absl/flags:flag",
        "@com_google_absl//absl/flags:parse",
    ],
)