OAuth2 Callback not hitting handler - asp.net-mvc

Question

When the server I'm authenticating with responds with the redirect back to my callback URL, my code doesn't seem to know how to handle the request - as if I'm missing a handler.
I thought the CallbackPath property would wire this up and pass the incoming request to my handler, is that not correct?
I just get a 404 from my app, the URL has the code=XXXX etc.
Here's my code
public Startup(IConfiguration configuration)
{
Configuration = configuration;
}
public IConfiguration Configuration { get; }
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
string authScheme = CookieAuthenticationDefaults.AuthenticationScheme;
services.AddAuthentication(o =>
{
o.DefaultChallengeScheme = CookieAuthenticationDefaults.AuthenticationScheme;
o.DefaultSignInScheme = CookieAuthenticationDefaults.AuthenticationScheme;
o.DefaultAuthenticateScheme = "MyAuthScheme";
o.DefaultScheme = CookieAuthenticationDefaults.AuthenticationScheme;
}).AddOAuth(authScheme, options =>
{
options.ClientId = "xxxxxxxxxxxxxxxxxxxxxxxx";
options.AuthorizationEndpoint = "https://example.com/adfs/oauth2/authorize?resource=https://example.net/";
options.TokenEndpoint = "https://example.com/adfs/oauth2/token";
options.CallbackPath = new Microsoft.AspNetCore.Http.PathString("/signin-myauthserver");
options.ClientSecret = "xxxxxxxxxxxxxxxxxxxxxxxx";
});
services.AddMvc().AddJsonOptions(options => options.SerializerSettings.ContractResolver = new DefaultContractResolver());
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IHostingEnvironment env)
{
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
app.UseBrowserLink();
}
else
{
app.UseExceptionHandler("/Error");
}
app.UseAuthentication();
app.UseStaticFiles();
app.UseMvc(routes =>
{
routes.MapRoute(
name: "default",
template: "{controller}/{action=Index}/{id?}");
});
}