demotool/shaders.cc

#include "externals.hh"
#include "shaders.hh"
#include "globals.hh"


namespace {

const std::regex PreprocDirective_( "^\\s*//!\\s*([a-z]+(\\s+([^\\s]+))*)\\s*$" );

const std::regex GLSLErrorNv_( "^[0-9]*\\(([0-9]+).*$" );

const std::map< std::string , E_ShaderInput > InputTypes_( ([] {
	std::map< std::string , E_ShaderInput > t;
	t.emplace( "chunk" , E_ShaderInput::CHUNK );
	t.emplace( "library" , E_ShaderInput::LIBRARY );
	t.emplace( "lib" , E_ShaderInput::LIBRARY );
	t.emplace( "vertex" , E_ShaderInput::VERTEX );
	t.emplace( "fragment" , E_ShaderInput::FRAGMENT );
	t.emplace( "compute" , E_ShaderInput::COMPUTE );
	t.emplace( "geo" , E_ShaderInput::GEOMETRY );
	t.emplace( "geometry" , E_ShaderInput::GEOMETRY );
	return t;
})());

const GLenum ProgramTypes_[] = {
	GL_VERTEX_SHADER ,
	GL_FRAGMENT_SHADER ,
	GL_GEOMETRY_SHADER ,
	GL_COMPUTE_SHADER
};

const GLbitfield PipelineStages_[] = {
	GL_VERTEX_SHADER_BIT ,
	GL_FRAGMENT_SHADER_BIT ,
	GL_GEOMETRY_SHADER_BIT ,
	GL_COMPUTE_SHADER_BIT
};

static_assert( sizeof( PipelineStages_ ) / sizeof( GLbitfield ) == size_t( E_ShaderType::__COUNT__ ) ,
		"missing pipeline stage constants" );


/*============================================================================*/

// Input reader state and functions, used when loading a source file
struct T_InputReader_
{
	using T_Tokens_ = std::vector< std::string >;

	FILE* const file;
	T_ShaderInput& input;
	uint32_t line{ 0 };

	char* buffer{ nullptr };
	ssize_t readCount;

	std::string accumulator{ };
	uint32_t accumLines{ 0 };

	T_InputReader_( __rd__ FILE* const file ,
			__rw__ T_ShaderInput& input )
		: file( file ) , input( input )
	{ }
	~T_InputReader_( );

	void read( );
	void nl( );
	void addAccumulated( );
	void handleDirective(
			__rd__ T_Tokens_ const& tokens );
	void error( __rd__ std::string const& err );
};

/*----------------------------------------------------------------------------*/

T_InputReader_::~T_InputReader_( )
{
	if ( buffer ) {
		free( buffer );
	}
	fclose( file );
}

void T_InputReader_::read( )
{
	size_t bsz( 0 );
	while ( ( readCount = getline( &buffer , &bsz , file ) ) != -1 ) {
		line ++;
		std::cmatch match;
		if ( std::regex_match( buffer , match , PreprocDirective_ ) ) {
			const T_Tokens_ tokens( ([](std::string const& a) {
				using stri = std::istream_iterator< std::string >;
				std::istringstream iss( a );
				return T_Tokens_{ stri( iss ) , stri( ) };
			})( match[ 1 ].str( ) ) );
			assert( tokens.size( ) >= 1 );
			handleDirective( tokens );
		} else {
			accumulator += buffer;
			accumLines ++;
		}
	}
	addAccumulated( );
}

void T_InputReader_::nl( )
{
	accumLines ++;
	accumulator += '\n';
}

void T_InputReader_::addAccumulated( )
{
	if ( accumLines ) {
		auto& ck( input.chunks );
		ck.emplace_back( E_ShaderInputChunk::CODE ,
				std::move( accumulator ) , accumLines );
		accumulator = {};
		accumLines = 0;
	}
}

void T_InputReader_::handleDirective(
		__rd__ T_Tokens_ const& tokens )
{
	auto const& directive( tokens[ 0 ] );

	if ( directive == "include" ) {
		if ( tokens.size( ) != 2 ) {
			nl( );
			error( "invalid arguments" );
			return;
		}
		addAccumulated( );
		auto& ck( input.chunks );
		ck.emplace_back( E_ShaderInputChunk::INCLUDE , tokens[ 1 ] , 1 );

	} else if ( directive == "type" ) {
		nl( );
		if ( tokens.size( ) != 2 ) {
			error( "invalid arguments" );
			return;
		}
		auto pos( InputTypes_.find( tokens[ 1 ] ) );
		if ( pos == InputTypes_.end( ) ) {
			error( "unknown type" );
		} else {
			input.type = pos->second;
		}

	} else {
		nl( );
		error( "unknown directive" );
	}
}

void T_InputReader_::error(
		__rd__ std::string const& err )
{
	input.errors.push_back( T_ShaderInputError{
			line , err } );
}

} // namespace


/*= T_ShaderInput ============================================================*/

bool T_ShaderInput::load(
		__rd__ std::string const& path )
{
	type = E_ShaderInput::CHUNK;
	chunks.clear( );
	errors.clear( );

	FILE* const file{ fopen( path.c_str( ) , "r" ) };
	if ( !file ) {
		return false;
	}

	T_InputReader_ reader( file , *this );
	reader.read( );

	return true;
}


/*= T_ShaderPipeline =========================================================*/

T_ShaderPipeline::T_ShaderPipeline( )
	: T_ShaderPipeline( -1 )
{ }

T_ShaderPipeline::T_ShaderPipeline(
		__rd__ T_ShaderPipeline const& other )
	: T_ShaderPipeline( other.smIndex_ )
{ }

T_ShaderPipeline::T_ShaderPipeline(
		__rw__ T_ShaderPipeline&& other ) noexcept
	: T_ShaderPipeline( )
{
	std::swap( other.smIndex_ , smIndex_ );
}

T_ShaderPipeline::T_ShaderPipeline(
		__rd__ const int32_t index )
	: smIndex_( index )
{
	if ( smIndex_ >= 0 ) {
		Globals::Shaders( ).pipelines_[ smIndex_ ].references ++;
	}
}

T_ShaderPipeline::~T_ShaderPipeline( )
{
	if ( smIndex_ >= 0 ) {
		Globals::Shaders( ).dereferencePipeline( smIndex_ );
	}
}

T_ShaderPipeline& T_ShaderPipeline::operator=(
		__rd__ T_ShaderPipeline const& other )
{
	if ( this != &other ) {
		if ( smIndex_ >= 0 ) {
			Globals::Shaders( ).dereferencePipeline( smIndex_ );
		}
		smIndex_ = other.smIndex_;
		if ( smIndex_ >= 0 ) {
			Globals::Shaders( ).pipelines_[ smIndex_ ].references ++;
		}
	}
	return *this;
}

T_ShaderPipeline& T_ShaderPipeline::operator=(
		__rw__ T_ShaderPipeline&& other ) noexcept
{
	if ( this != &other ) {
		if ( smIndex_ >= 0 ) {
			Globals::Shaders( ).dereferencePipeline( smIndex_ );
		}
		smIndex_ = other.smIndex_;
		other.smIndex_ = -1;
	}
	return *this;
}

bool T_ShaderPipeline::valid( ) const noexcept
{
	return smIndex_ >= 0
		&& Globals::Shaders( ).pipelines_[ smIndex_ ].id != 0;
}

void T_ShaderPipeline::enable( ) const
{
	if ( valid( ) ) {
		glBindProgramPipeline( Globals::Shaders( ).pipelines_[ smIndex_ ].id );
	}
}

GLuint T_ShaderPipeline::program(
		__rd__ const E_ShaderType program ) const
{
	if ( !valid( ) ) {
		return 0;
	}

	auto const& sm( Globals::Shaders( ) );
	auto const& pl( sm.pipelines_[ smIndex_ ] );
	for ( auto const& pn : pl.programs ) {
		auto pos( sm.programIndex_.find( pn ) );
		if ( pos == sm.programIndex_.end( ) ) {
			continue;
		}
		auto& p( sm.programs_[ pos->second ] );
		if ( p.code.type == program ) {
			return p.id;
		}
	}
	return 0;
}


/*= T_CodeBuilder_ ===========================================================*/

namespace {

using F_GetInput_ = std::function< T_ShaderInput const*( std::string const& ) >;
using T_Code_ = T_ShaderManager::T_ShaderCode;

// Code builder, state and functions
struct T_CodeBuilder_
{
	struct T_StackEntry_ {
		std::string name;
		T_ShaderInput const* input;
		uint32_t pos;
	};

	F_GetInput_ loader;
	const std::string name;
	T_Code_& code;

	T_ShaderInput const* main;
	std::map< std::string , uint32_t > pos;
	std::vector< T_StackEntry_ > stack;
	std::set< std::string > libraries;
	T_ShaderInput const* current;
	uint32_t cpos{ 0 };
	std::string cname;

	T_CodeBuilder_( __rd__ F_GetInput_ loader ,
			__rd__ std::string const& name ,
			__rw__ T_Code_& code )
		: loader( loader ) , name( name ) , code( code ) ,
			main( loader( name ) )
	{ }

	bool buildCode( );
	void appendChunk( __rd__ T_ShaderInputChunk const& chunk );
	void include( __rd__ std::string const& name ,
			__rd__ const uint32_t lines );
	void next( );
};

/*----------------------------------------------------------------------------*/

bool T_CodeBuilder_::buildCode( )
{
	code = T_Code_{ };
	code.files.emplace( name , main != nullptr );
	if ( !main ) {
		return false;
	}
	if ( main->type == E_ShaderInput::CHUNK
			|| main->type == E_ShaderInput::LIBRARY ) {
		code.errors.push_back( T_ShaderError{
				name , 0 , "invalid type" } );
	} else {
		code.type = E_ShaderType( int( main->type ) - 2 );
	}
	cname = name;
	current = main;

	while ( cpos < current->chunks.size( ) ) {
		auto& chunk( current->chunks[ cpos ] );
		if ( chunk.type == E_ShaderInputChunk::CODE ) {
			appendChunk( chunk );
		} else {
			include( chunk.text , chunk.lines );
		}

		next( );
	}

	return true;
}

void T_CodeBuilder_::appendChunk(
		__rd__ T_ShaderInputChunk const& chunk )
{
	code.sources.push_back( cname );
	code.counts.push_back( chunk.lines );
	code.starts.push_back( pos[ cname ] );
	code.code += chunk.text;
	pos[ cname ] += chunk.lines;
}

void T_CodeBuilder_::include(
		__rd__ std::string const& nname ,
		__rd__ const uint32_t lines )
{
	const auto prevPos( pos[ cname ] );
	pos[ cname ] += lines;

	// Avoid recursion
	if ( cname == nname || 0 != count_if( stack.begin( ) , stack.end( ) ,
				[nname] ( T_StackEntry_ const& e ) {
					return nname == e.name;
				} ) ) {
		code.errors.push_back( T_ShaderError{ cname , prevPos ,
				"recursive inclusion of '" + nname + "'" } );
		return;
	}

	// Avoid including libraries more than once
	if ( libraries.find( nname ) != libraries.end( ) ) {
		return;
	}

	T_ShaderInput const* const isi( loader( nname ) );
	code.files.emplace( nname , isi != nullptr );

	// Check for problems
	if ( !isi ) {
		// Not found
		code.errors.push_back( T_ShaderError{ cname , prevPos ,
				"file not found" } );
		return;
	}
	if ( isi->type != E_ShaderInput::CHUNK && isi->type != E_ShaderInput::LIBRARY ) {
		// Trying to load a top-level shader
		code.errors.push_back( T_ShaderError{ cname , prevPos ,
				"trying to include a top-level file" } );
		return;
	}

	// Add input loader errors
	if ( libraries.find( nname ) == libraries.end( ) ) {
		for ( auto const& errs : isi->errors ) {
			code.errors.push_back( T_ShaderError{
					nname , errs.line , errs.error } );
		}
	}
	libraries.insert( nname );

	// Enter the new file
	stack.push_back( T_StackEntry_{ cname , current , cpos } );
	cname = nname;
	current = isi;
	cpos = UINT32_MAX;
	pos[ cname ] = 0;
}

void T_CodeBuilder_::next( )
{
	cpos ++;
	while ( cpos == current->chunks.size( ) && !stack.empty( ) ) {
		T_StackEntry_ const& se( stack[ stack.size( ) - 1 ] );
		pos.erase( cname );
		cpos = se.pos + 1;
		current = se.input;
		cname = se.name;
		stack.pop_back( );
	}
}


} // namespace

/*============================================================================*/


T_ShaderPipeline T_ShaderManager::pipeline(
		__rd__ std::initializer_list< std::string > shaders )
{
	if ( shaders.size( ) < 1 || shaders.size( ) > 5 ) {
		return {};
	}

	const std::string id( ([&shaders] () {
		std::ostringstream oss;
		std::copy( shaders.begin( ) , shaders.end( ) ,
				std::ostream_iterator< std::string >( oss , "|" ) );
		return oss.str( );
	} )() );
	auto pos( pipelineIndex_.find( id ) );
	if ( pos != pipelineIndex_.end( ) ) {
		return T_ShaderPipeline{ int32_t( pos->second ) };
	}

	const auto index( newPipelineRecord( ) );
	auto& p( pipelines_[ index ] );
	pipelineIndex_.emplace( id , index );
	p.idString = id;
	p.id = 0;
	p.references = 0;
	p.programs = shaders;
	for ( auto const& pName : shaders ) {
		loadProgram( index , pName );
	}
	initPipeline( p );

	return T_ShaderPipeline{ int32_t( index ) };
}


void T_ShaderManager::update( )
{
	inputs_.clear( );

	// Check for missing files
	for ( auto it = missing_.begin( ) ; it != missing_.end( ) ; ++ it ) {
		const bool exists( ([] ( std::string const& name ) -> bool {
			struct stat buffer;
			return ( stat( name.c_str( ) , &buffer ) == 0 );
		})( "shaders/" + (*it).first ) );
		if ( !exists ) {
			continue;
		}
		updates_.insert( (*it).second.begin( ) , (*it).second.end( ) );
		missing_.erase( (*it).first );
	}

	// Reset programs that need to be updated
	std::set< uint32_t > pipelines;
	for ( auto const& name : updates_ ) {
		auto p( programIndex_.find( name ) );
		if ( p == programIndex_.end( ) ) {
			updates_.erase( name );
		} else {
			auto& pr( programs_[ p->second ] );
			pipelines.insert( pr.references.begin( ) , pr.references.end( ) );
			resetProgram( pr );
		}
	}
	if ( updates_.empty( ) ) {
		return;
	}

	// Reset pipelines affected by the programs above
	for ( auto plid : pipelines ) {
		auto& pipeline( pipelines_[ plid ] );
		if ( pipeline.id != 0 ) {
			glDeleteProgramPipelines( 1 , &pipeline.id );
			pipeline.id = 0;
		}
	}

	// Try to load all updated programs
	for ( auto const& name : updates_ ) {
		auto& pr( programs_[ programIndex_[ name ] ] );
		initProgram( pr );
		for ( auto const& e : pr.code.errors ) {
			printf( "%s:%d: %s\n" , e.source.c_str( ) , e.line ,
					e.error.c_str( ) );
		}
	}

	// Try to initialise all affected pipelines
	for ( auto plid : pipelines ) {
		initPipeline( pipelines_[ plid ] );
	}

	updates_.clear( );
}


uint32_t T_ShaderManager::newPipelineRecord( )
{
	uint32_t i = 0;
	while ( i < pipelines_.size( ) ) {
		if ( pipelines_[ i ].references == 0 ) {
			return i;
		}
		i ++;
	}
	pipelines_.emplace_back( );
	return i;
}

uint32_t T_ShaderManager::newProgramRecord( )
{
	uint32_t i = 0;
	while ( i < programs_.size( ) ) {
		if ( programs_[ i ].references.empty( ) ) {
			return i;
		}
		i ++;
	}
	programs_.emplace_back( );
	return i;
}


void T_ShaderManager::loadProgram(
		__rd__ const uint32_t pipeline ,
		__rd__ std::string const& name )
{
	if ( useExistingProgram( pipeline , name ) ) {
		return;
	}

	const uint32_t index( newProgramRecord( ) );
	auto& program( programs_[ index ] );
	programIndex_.emplace( name , index );
	program.name = name;
	program.references.insert( pipeline );
	program.id = 0;
	initProgram( program );
	for ( auto const& e : program.code.errors ) {
		printf( "%s:%d: %s\n" , e.source.c_str( ) , e.line , e.error.c_str( ) );
	}
}

bool T_ShaderManager::useExistingProgram(
		__rd__ const uint32_t pipeline ,
		__rd__ std::string const& name )
{
	auto pos( programIndex_.find( name ) );
	if ( pos == programIndex_.end( ) ) {
		return false;
	}
	auto& refs( programs_[ pos->second ].references );
	assert( refs.find( pipeline ) == refs.end( ) );
	refs.insert( pipeline );
	return true;
}


T_ShaderInput const* T_ShaderManager::getInput(
		__rd__ std::string const& name )
{
	auto pos( inputs_.find( name ) );
	if ( pos != inputs_.end( ) ) {
		return pos->second.get( );
	}
	T_ShaderInput ni;
	if ( !ni.load( "shaders/" + name ) ) {
		return nullptr;
	}
	inputs_.emplace( name , std::make_unique< T_ShaderInput >( std::move( ni ) ) );
	return inputs_.find( name )->second.get( );
}


void T_ShaderManager::dereferencePipeline(
		__rd__ const uint32_t index )
{
	auto& pipeline( pipelines_[ index ] );
	assert( pipeline.references > 0 );
	pipeline.references --;
	if ( pipeline.references > 0 ) {
		return;
	}

	pipelineIndex_.erase( pipeline.idString );
	if ( pipeline.id != 0 ) {
		glDeleteProgramPipelines( 1 , &pipeline.id );
		pipeline.id = 0;
	}

	for ( auto const& pName : pipeline.programs ) {
		auto pos( programIndex_.find( pName ) );
		assert( pos != programIndex_.end( ) );
		dereferenceProgram( pos->second , index );
	}
}


void T_ShaderManager::dereferenceProgram(
		__rd__ const uint32_t index ,
		__rd__ const uint32_t pipeline )
{
	auto& program( programs_[ index ] );
	auto& refs( program.references );
	assert( refs.find( pipeline ) != refs.end( ) );
	refs.erase( pipeline );
	if ( refs.size( ) != 0 ) {
		return;
	}
	resetProgram( program );
	programIndex_.erase( program.name );
}


void T_ShaderManager::initPipeline(
		__rw__ T_Pipeline_& pipeline ) const
{
	assert( pipeline.id == 0 );

	printf( "init pipeline %s\n" , pipeline.idString.c_str( ) );
	constexpr auto nst{ size_t( E_ShaderType::__COUNT__ ) };
	int32_t programs[ nst ];
	for ( auto i = 0u ; i < nst ; i ++ ) {
		programs[ i ] = -1;
	}

	for ( auto const& pName : pipeline.programs ) {
		const uint32_t pid( programIndex_.find( pName )->second );
		auto const& program( programs_[ pid ] );
		if ( programs[ int( program.code.type ) ] != -1
				|| program.id == 0 ) {
			printf( "... failed\n" );
			return;
		}
		programs[ int( program.code.type ) ] = pid;
	}

	GLuint id( 0 );
	glGenProgramPipelines( 1 , &id );
	GL_CHECK( return );
	for ( auto i = 0u ; i < nst ; i ++ ) {
		const auto pid( programs[ i ] );
		if ( pid == -1 ) {
			continue;
		}
		auto& program( programs_[ pid ] );
		const GLbitfield type( PipelineStages_[ int( program.code.type ) ] );
		glUseProgramStages( id , type , program.id );
	}
	GL_CHECK({
		glDeleteProgramPipelines( 1 , &id );
		printf( "... failed\n" );
		return;
	});

	pipeline.id = id;
	printf( "... success\n" );
}


void T_ShaderManager::initProgram(
		__rw__ T_Program_& program )
{
	assert( program.id == 0 );

	// Build the code
	auto name( program.name );
	printf( "init program %s\n" , program.name.c_str( ) );
	auto& code( program.code );
	T_CodeBuilder_ cb( [this]( std::string const& n ) { return getInput( n ); } ,
			name , code );
	const bool built(
			T_CodeBuilder_{
				[this]( std::string const& n ) { return getInput( n ); } ,
				program.name , code
			}.buildCode( ) );

	// Initialise file watcher + missing files
	program.watch = std::make_unique< T_WatchedFiles >( Globals::Watcher( ) ,
			[this,name]() {
				programUpdated( name );
			} );
	for ( auto entry : code.files ) {
		if ( entry.second ) {
			program.watch->watch( "shaders/" + entry.first );
		} else {
			missing_[ entry.first ].insert( name );
		}
	}

	if ( !( built && code.errors.empty( ) ) ) {
		return;
	}

	// Try to compile the program
	char const* const list[] = {
		program.code.code.c_str( )
	};
	const GLenum sid = glCreateShaderProgramv(
			ProgramTypes_[ int( code.type ) ] ,
			1 , &list[ 0 ] );
	if ( sid == 0 ) {
		code.errors.push_back( T_ShaderError{
				name , 0 , "failed to create GL program" } );
		return;
	}

	// Read and convert the log
	int infoLogLength( 0 );
	glGetProgramiv( sid , GL_INFO_LOG_LENGTH , &infoLogLength );
	if ( infoLogLength ) {
		char buffer[ infoLogLength + 1 ];
		glGetProgramInfoLog( sid , infoLogLength , nullptr , buffer );
		char* start( buffer );
		char* found( strchr( buffer , '\n' ) );
		while ( found ) {
			*found = 0;
			parseGLSLError( code , start );
			start = found + 1;
			found = strchr( start , '\n' );
		}
		if ( start < &buffer[ infoLogLength - 1 ] ) {
			parseGLSLError( code , start );
		}
	}

	// Did it link?
	GLint lnk( 0 );
	glGetProgramiv( sid , GL_LINK_STATUS , &lnk );
	if ( lnk ) {
		program.id = sid;
		printf( "... success\n" );
	} else {
		glDeleteProgram( sid );
	}
}

void T_ShaderManager::parseGLSLError(
		__rw__ T_ShaderCode& code ,
		__rd__ char const* errorLine )
{
	assert( !code.sources.empty( ) );

	std::cmatch m;
	uint32_t rawLine;
	if ( std::regex_match( errorLine , m , GLSLErrorNv_ ) ) {
		rawLine = atoi( m[ 1 ].str( ).c_str( ) );
	} else {
		rawLine = 0;
	}

	uint32_t check = 0 , pos = 0;
	while ( pos < code.starts.size( ) && check < rawLine ) {
		check += code.counts[ pos ];
		pos ++;
	}

	code.errors.push_back( T_ShaderError{
			pos == 0 ? "*unknown*" : code.sources[ pos - 1 ] ,
			pos == 0 ? 0 : ( rawLine + code.counts[ pos - 1 ] - check + code.starts[ pos - 1 ] ) ,
			errorLine } );
}

void T_ShaderManager::programUpdated(
		__rd__ std::string const& name )
{
	updates_.insert( name );
}

void T_ShaderManager::resetProgram(
		__rw__ T_Program_& program )
{
	if ( program.watch ) {
		program.watch.reset( );
	}
	if ( program.id != 0 ) {
		glDeleteProgram( program.id );
		program.id = 0;
		return;
	}

	for ( auto entry : program.code.files ) {
		if ( entry.second || missing_.find( entry.first ) == missing_.end( ) ) {
			continue;
		}
		auto& set( missing_[ entry.first ] );
		set.erase( program.name );
		if ( set.empty( ) ) {
			missing_.erase( entry.first );
		}
	}
}