The problem here is that IQueryable is an action filter as well and actually runs after your custom filter. Unfortuantely there is no support for ordering filters at the granularity you require.

However, you can hack into the system and abuse it and get it to work for your case by writing a custom filter provider and tweaking the FiletrScope enum a little bit. Same sample code follows,

public class MyActionFilterProvider : IFilterProvider
{
    public System.Collections.Generic.IEnumerable<Filter> GetFilters(HttpConfiguration configuration, System.Web.Http.Controllers.HttpActionDescriptor actionDescriptor)
    {
        if (actionDescriptor.ReturnType == typeof(IQueryable<char>))
        {
            return new[] { new Filter(new MyActionFilter(), (FilterScope)500) };
        }
        else
        {
            return Enumerable.Empty<Filter>();
        }
    }
}

public class MyActionFilter : ActionFilterAttribute
{
    public override void OnActionExecuting(System.Web.Http.Controllers.HttpActionContext actionContext)
    {
        base.OnActionExecuting(actionContext);
    }
}

public class QueryableController : ApiController
{
    public IQueryable<char> GetStrings()
    {
        return "hello, world".ToArray().AsQueryable();
    }
}