Use Authorization middleware instead of AuthorizationAttribute ASPNET Core

I have a dedicated IdServer running that has the login page that other applications will boot unauthenticated users to.

My current pipeline is:

app.UseCookieAuthentication
app.UseOpenIdConnectAuthentication
app.UseDefaultFiles // because it is a SPA app
app.UseStaticFiles // the SPA app

So all tutorials say to use [Authorize] on your controllers...

However, I want middle to authorize all of my controllers, and static files.

So how do I write a middleware to handle that.

My current setup is:

public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerFactory loggerFactory, IOptions<IdentityServerAppOptions> identityServerAppOptions)
{
    loggerFactory.AddConsole(Configuration.GetSection("Logging"));
    loggerFactory.AddDebug();

    var serverAppOptions = identityServerAppOptions.Value;

    loggerFactory.CreateLogger("Configure").LogDebug("Identity Server Authority Configured: {0}", serverAppOptions.Authority);

    JwtSecurityTokenHandler.DefaultInboundClaimTypeMap.Clear();
    app.UseCookieAuthentication(new CookieAuthenticationOptions
    {
        AuthenticationScheme = "Cookies"
    });
    app.UseOpenIdConnectAuthentication(new OpenIdConnectOptions
    {
        AuthenticationScheme = "oidc",
        SignInScheme = "Cookies",

        Authority = serverAppOptions.Authority,
        RequireHttpsMetadata = false,

        ClientId = "Video",
        SaveTokens = true
    });

    app.Use(async (context, next) =>
    {
        var authService = context.RequestServices.GetRequiredService<IAuthorizationService>();


        if (!await authService.AuthorizeAsync(context.User, context, "Api"))
        {
            // This is as far as I have got, here we should boot them to IdServer
        }
    });

    app.UseDefaultFiles(new DefaultFilesOptions
    {
        DefaultFileNames = new List<string> { "index.html" },
        RequestPath = new PathString("")
    });
    app.UseStaticFiles(new StaticFileOptions
    {
        OnPrepareResponse = ctx =>
        {
            ctx.Context.Response.Headers.Append("Cache-Control", "no-cache");
        }
    });
    app.UseMvc();
}