/**
 * Copyright 2005-2007 ECMWF
 *
 * Licensed under the GNU Lesser General Public License which
 * incorporates the terms and conditions of version 3 of the GNU
 * General Public License.
 * See LICENSE and gpl-3.0.txt for details.
 */

/*
 *
 * Description: index
 *
 * Author: Enrico Fucile <enrico.fucile@ecmwf.int>
 *
 *
 */

#include "grib_api_internal.h"

#define UNDEF_LONG -99999
#define UNDEF_DOUBLE -99999

#define NULL_MARKER 0
#define NOT_NULL_MARKER 255

static const char* mars_keys =
    "mars.date,mars.time,mars.expver,mars.stream,mars.class,mars.type,"
	"mars.step,mars.param,mars.levtype,mars.levelist,mars.number,mars.iteration,"
	"mars.domain,mars.fcmonth,mars.fcperiod,mars.hdate,mars.method,"
	"mars.model,mars.origin,mars.quantile,mars.range,mars.refdate,mars.direction,mars.frequency";


static int grib_filesid=0;
static int index_count;

static char* get_key(char** keys,int *type) {
	char* key=NULL;
	char* p=NULL;

	if (*keys == 0 || keys==NULL) return NULL;
	*type=GRIB_TYPE_UNDEFINED;
	p=*keys;
	while (*p == ' ') p++;


	while (*p != 0 && *p != ':' && *p != ',') p++;
	if ( *p == ':' ) {
		*type=grib_type_to_int(*(p+1));
		*p=0;
		p++;
		while (*p != 0 && *p != ',') {*(p++)=0;}
	} else *type=GRIB_TYPE_UNDEFINED;
	if (*p) {*p=0;p++;}
	key=*keys;
	*keys= *p==0 ? NULL : p;
	return key;
}

static int compare_long(const void* a,const void* b) {
	long* arg1 = (long*) a;
	long* arg2 = (long*) b;
    if( *arg1 == *arg2 ) return 0;

    return *arg1 < *arg2 ? -1 : 1;
}

static int compare_double(const void* a,const void* b) {
	double* arg1 = (double*) a;
	double* arg2 = (double*) b;
    if( *arg1 == *arg2 ) return 0;

    return *arg1 < *arg2 ? -1 : 1;

}


static int compare_string(const void* a,const void* b) {
  char* arg1 = *(char* const*) a;
  char* arg2 = *(char* const*) b;

  while (*arg1 != 0 && *arg2 != 0 && *arg1 == *arg2 ) {arg1++;arg2++;}
  
  if( *arg1 == *arg2 ) return 0;

  return *arg1 < *arg2 ? -1 : 1;
}


static int grib_index_keys_compress(grib_context* c,grib_index* index,int* compress) {
	grib_index_key *keys=index->keys->next;
	grib_index_key *prev=index->keys;
	int level=0;

	if (!keys) return 0;

	level=1;
	while (keys) {
		if (keys->values_count==1) {
			prev->next=keys->next;
			grib_context_free(c,keys->name);
			grib_context_free(c,keys);
			keys=prev->next;
			compress[level]=1;
			level++;
		} else {
			prev=keys;
			keys=keys->next;
			compress[level]=0;
			level++;
		}
	}

	if (index->keys->values_count==1) {
		keys=index->keys;
		index->keys=index->keys->next;
		grib_context_free(c,keys->name);
		grib_context_free(c,keys);
		compress[0]=1;
	} else compress[0]=0;

	return 0;
}

static int grib_index_fields_compress(grib_context* c,
		grib_field_tree* fields,grib_field_tree* prev,int level,int* compress) {

	if (!fields) return 0;

	if (!prev) {
		if (fields->next) 
			grib_index_fields_compress(c,fields->next,0,level,compress);
		level++;
		return grib_index_fields_compress(c,fields->next_level,fields,level,compress);
	}

	if (compress[level]) {
		if (!fields->next_level) prev->field=fields->field;

		prev->next_level=fields->next_level;
		grib_context_free(c,fields->value);
		grib_context_free(c,fields);
		level++;
		grib_index_fields_compress(c,prev->next_level,prev,level,compress);
	} else {
		grib_field_tree* next;
		next=fields->next;
		level++;
		while (next) { 
			grib_index_fields_compress(c,next->next_level,next,level,compress);
			next=next->next;
		}
		grib_index_fields_compress(c,fields->next_level,fields,level,compress);
	}

	return 0;
}

int grib_index_compress(grib_index* index) {
	int err=0;
	grib_context* c=index->context;
	int compress[200]={0,};
	
	if (!index->keys->next) return 0;

	err=grib_index_keys_compress(c,index,compress);
	if (err) return err;

	grib_index_fields_compress(c,index->fields,0,0,compress);

	if (!index->fields->next) {
		grib_field_tree* next_level=index->fields->next_level;
		grib_context_free(c,index->fields->value);
		grib_context_free(c,index->fields);
		index->fields=next_level;
	}
	return 0;
}

static grib_index_key* grib_index_new_key(grib_context* c,grib_index_key* keys,
		const char* key,int type,int *err) {
	grib_index_key *next=NULL,*current=NULL;
	grib_string_list* values=NULL;

	next=(grib_index_key*)grib_context_malloc_clear(c,sizeof(grib_index_key));
	if (!next) {
		grib_context_log(c,GRIB_LOG_ERROR,
				"unable to allocate %d bytes",
				sizeof(grib_index_key));
		*err=GRIB_OUT_OF_MEMORY;
		return NULL;
	}
	values=grib_context_malloc_clear(c,sizeof(grib_string_list));
	if (!values) {
		grib_context_log(c,GRIB_LOG_ERROR,
				"unable to allocate %d bytes",
				sizeof(grib_string_list));
		*err=GRIB_OUT_OF_MEMORY;
		return NULL;
	}

	next->values=values;

	if (!keys) {
		keys=next;
		current=keys;
	} else {
		current=keys;
		while (current->next) current=current->next;
		current->next=next;
		current=current->next;
	}

	current->type=type;
	current->name=grib_context_strdup(c,key);
	return keys;
}

int grib_read_uchar( FILE* fh,unsigned char *val) {
	if (fread(val,sizeof(unsigned char),1,fh) <1) {
		if (feof(fh)) return GRIB_END_OF_FILE; 
		else return GRIB_IO_PROBLEM;
	}
	return GRIB_SUCCESS;
}

int grib_read_short( FILE* fh,short *val) {
	if (fread(val,sizeof(short),1,fh) <1) {
		if (feof(fh)) return GRIB_END_OF_FILE; 
		else return GRIB_IO_PROBLEM;
	}
	return GRIB_SUCCESS;
}

int grib_read_long( FILE* fh,long *val) {
	if (fread(val,sizeof(long),1,fh) <1) {
		if (feof(fh)) return GRIB_END_OF_FILE; 
		else return GRIB_IO_PROBLEM;
	}
	return GRIB_SUCCESS;
}

int grib_read_unsigned_long( FILE* fh,unsigned long *val) {
	if (fread(val,sizeof(long),1,fh) <1) {
		if (feof(fh)) return GRIB_END_OF_FILE;
		else return GRIB_IO_PROBLEM;
	}
	return GRIB_SUCCESS;
}

int grib_write_uchar(FILE* fh,unsigned char val) {
	if (fwrite(&val,sizeof(unsigned char),1,fh)<1)
		return GRIB_IO_PROBLEM;
	return GRIB_SUCCESS;
}

int grib_write_short(FILE* fh,short val) {
	if (fwrite(&val,sizeof(short),1,fh)<1)
		return GRIB_IO_PROBLEM;
	return GRIB_SUCCESS;
}

int grib_write_long(FILE* fh,long val) {
	if (fwrite(&val,sizeof(long),1,fh)<1)
		return GRIB_IO_PROBLEM;
	return GRIB_SUCCESS;
}

int grib_write_unsigned_long(FILE* fh,unsigned long val) {
	if (fwrite(&val,sizeof(long),1,fh)<1)
		return GRIB_IO_PROBLEM;
	return GRIB_SUCCESS;
}

int grib_write_string(FILE* fh,const char* s)
{
	size_t len = strlen(s);
	grib_write_uchar(fh,(unsigned char)len);
	if (fwrite(s,1,len,fh)<len)
		return GRIB_IO_PROBLEM;
	return GRIB_SUCCESS;
}

int grib_write_identifier(FILE* fh) {
    return grib_write_string(fh,"GRBIDX1");
}

int grib_write_null_marker(FILE* fh) {
	return grib_write_uchar(fh,NULL_MARKER);
}

int grib_write_not_null_marker(FILE* fh) {
	return grib_write_uchar(fh,NOT_NULL_MARKER);
}


char *grib_read_string(grib_context* c,FILE* fh,int *err) {
	unsigned char len=0;
	char* s=NULL;
	*err = grib_read_uchar(fh,&len);

	if (*err) return NULL;
	s = grib_context_malloc_clear(c,len+1);
	if (fread(s,len,1,fh) < 1) 
	{
		if (feof(fh)) *err=GRIB_END_OF_FILE;
		else *err=GRIB_IO_PROBLEM;
		return NULL;
	}
	s[len] = 0;

	return s;
}

static int grib_write_field(FILE* fh,grib_field* field) {
	int err;
	if (!field) 
		return grib_write_null_marker(fh);

	err=grib_write_not_null_marker(fh);
	if (err) return err;

	err=grib_write_short(fh,field->file->id);
	if (err) return err;

	err=grib_write_unsigned_long(fh,field->offset);
	if (err) return err;

	err=grib_write_unsigned_long(fh,field->length);
	if (err) return err;

	err=grib_write_field(fh,field->next);
	if (err) return err;

	return GRIB_SUCCESS;
}

static grib_field* grib_read_field(grib_context* c,FILE* fh,grib_file** files,int *err) {

	grib_field* field=NULL;
	short file_id;
	unsigned char marker ;
	unsigned long offset;
	unsigned long length;
	
	*err = grib_read_uchar(fh,&marker);
	if(marker == NULL_MARKER) return NULL;
	if(marker != NOT_NULL_MARKER) {*err=GRIB_CORRUPTED_INDEX;return NULL;}

	index_count++;
	field=grib_context_malloc(c,sizeof(grib_field));
	*err=grib_read_short(fh,&file_id);
	if (*err) return NULL;

	field->file=files[file_id];

	*err=grib_read_unsigned_long(fh,&offset);
	field->offset=offset;
	if (*err) return NULL;

	*err=grib_read_unsigned_long(fh,&length);
	field->length=length;
	if (*err) return NULL;

	field->next=grib_read_field(c,fh,files,err);

	return field;
}

static int grib_write_field_tree(FILE* fh,grib_field_tree* tree) {

	int err=0;
	
	if(!tree)  
		return grib_write_null_marker(fh);

	err=grib_write_not_null_marker(fh);
	if (err) return err;
	
	err=grib_write_field(fh,tree->field);
	if (err) return err;
	
	err=grib_write_string(fh,tree->value);
	if (err) return err;
	
	err=grib_write_field_tree(fh,tree->next_level);
	if (err) return err;

	err=grib_write_field_tree(fh,tree->next);
	if (err) return err;
	return GRIB_SUCCESS;
}

grib_field_tree* grib_read_field_tree(grib_context* c, FILE* fh,
				grib_file** files,int *err) {

	grib_field_tree* tree=NULL;
	unsigned char marker=0;
	*err = grib_read_uchar(fh,&marker);

	if(marker == NULL_MARKER) return NULL;
    if(marker != NOT_NULL_MARKER) {*err=GRIB_CORRUPTED_INDEX;return NULL;}

	tree = grib_context_malloc(c, sizeof(grib_field_tree));
	tree->field = grib_read_field(c,fh,files,err);
	if (*err) return NULL;
	
	tree->value = grib_read_string(c,fh,err);
	if (*err) return NULL;
	
	tree->next_level = grib_read_field_tree(c,fh,files,err);
	if (*err) return NULL;

	tree->next = grib_read_field_tree(c,fh,files,err);
	if (*err) return NULL;

	return tree;
}


grib_index* grib_index_new(grib_context* c,const char* key,int *err) {
	grib_index* index;
	grib_index_key* keys=NULL;
	char* q;
	int type;
	char* p;

	if (!strcmp(key,"mars")) 
		return grib_index_new(c,mars_keys,err);
	
	p= grib_context_strdup(c,key);
	q=p;

	*err=0;
	if (!c) c=grib_context_get_default();

	index=(grib_index*)grib_context_malloc_clear(c,sizeof(grib_index));
	if (!index) {
		grib_context_log(c,GRIB_LOG_ERROR,"unable to create index");
		*err=GRIB_OUT_OF_MEMORY;
		return NULL;
	}
	index->context=c;

	while ((key=get_key(&p,&type))!=NULL) {
		keys=grib_index_new_key(c,keys,key,type,err);
		if (*err) return NULL;
	}
	index->keys=keys;
	index->fields=(grib_field_tree*)grib_context_malloc_clear(c,
			sizeof(grib_field_tree));
	if (!index->fields) {*err=GRIB_OUT_OF_MEMORY;return NULL;}

	grib_context_free(c,q);
	return index;
}

static void grib_index_values_delete(grib_context* c,grib_string_list* values) {

	if (!values) return;

	grib_index_values_delete(c,values->next);
	grib_context_free(c,values->value);
	grib_context_free(c,values);

	return ;
}

static void grib_index_key_delete(grib_context* c,grib_index_key* keys) {
	if (!keys) return;

	grib_index_key_delete(c,keys->next);

	grib_index_values_delete(c,keys->values);
	grib_index_values_delete(c,keys->current);
	grib_context_free(c,keys->name);
	grib_context_free(c,keys);

	return;
}


static long values_count=0;
static grib_string_list* grib_read_key_values(grib_context* c,FILE* fh,int *err) {
	grib_string_list* values;
	unsigned char marker=0;

	*err = grib_read_uchar(fh,&marker);
	if(marker == NULL_MARKER) return NULL;
    if(marker != NOT_NULL_MARKER) {*err=GRIB_CORRUPTED_INDEX;return NULL;}

	values_count++;

	values=grib_context_malloc_clear(c,sizeof(grib_string_list));
	values->value=grib_read_string(c,fh,err);
	if (*err) return NULL;

	values->next=grib_read_key_values(c,fh,err);
	if (*err) return NULL;

	return values;
}


static int grib_write_key_values(FILE* fh,grib_string_list* values) {
	int err=0;

	if (!values) 
		return grib_write_null_marker(fh);

	err=grib_write_not_null_marker(fh);
	if (err) return err;

	err=grib_write_string(fh,values->value);

	err=grib_write_key_values(fh,values->next);
	if (err) return err;

	return GRIB_SUCCESS;
}


static grib_index_key* grib_read_index_keys(grib_context* c,FILE* fh,int *err) {
	grib_index_key* keys=NULL;
	unsigned char marker=0;
	unsigned char type=0;
	
	if (!c) c=grib_context_get_default();

	*err = grib_read_uchar(fh,&marker);
	if(marker == NULL_MARKER) return NULL;
	if(marker != NOT_NULL_MARKER) {*err=GRIB_CORRUPTED_INDEX;return NULL;}

	keys= grib_context_malloc_clear(c,sizeof(grib_index_key));
	keys->name = grib_read_string(c,fh,err);
	if (*err) return NULL;

	*err = grib_read_uchar(fh,&type);
	keys->type=type;
	if (*err) return NULL;

	values_count=0;
	keys->values=grib_read_key_values(c,fh,err);
	if (*err) return NULL;

	keys->values_count=values_count;
	if (*err) return NULL;

	
	keys->next=grib_read_index_keys(c,fh,err);
	if (*err) return NULL;

	return keys;

}

static int grib_write_index_keys(FILE* fh,grib_index_key* keys) {
	int err=0;

	if (!keys) 
		return grib_write_null_marker(fh);

	err=grib_write_not_null_marker(fh);
	if (err) return err;

	err=grib_write_string(fh,keys->name);
	if (err) return err;

	err=grib_write_uchar(fh,(unsigned char)keys->type);
	if (err) return err;

	grib_write_key_values(fh,keys->values);

	err=grib_write_index_keys(fh,keys->next);
	if (err) return err;

	return GRIB_SUCCESS;
}

static void grib_field_delete(grib_context* c,grib_field* field) {
	int err=0;
	
	if (!field) return;

	grib_field_delete(c,field->next);

	grib_file_close(field->file->name,&err);
	field->file=NULL;

	grib_context_free(c,field);

	return;

}

static void grib_field_tree_delete(grib_context* c,grib_field_tree* tree) {

	if(!tree)  return;

	grib_field_delete(c,tree->field);
	grib_context_free(c,tree->value);
	
	grib_field_tree_delete(c,tree->next_level);

	grib_field_tree_delete(c,tree->next);

	grib_context_free(c,tree);

	return;
}

void grib_index_delete(grib_index* index) {
	grib_file* file=index->files;
	grib_index_key_delete(index->context,index->keys);
	grib_field_tree_delete(index->context,index->fields); 
	while (file) {
		grib_file* f=file;
		file=file->next;
		grib_file_delete(f);
	}
	grib_context_free(index->context,index);
}

static int grib_write_files(FILE* fh,grib_file* files) {
	int err;
	if (!files)
	       return grib_write_null_marker(fh);

    err=grib_write_not_null_marker(fh);
    if (err) return err;

	err=grib_write_string(fh,files->name);
    if (err) return err;

    err=grib_write_short(fh,(short)files->id);
    if (err) return err;

    return grib_write_files(fh,files->next);
}

static grib_file* grib_read_files(grib_context *c,FILE* fh,int *err) {
	unsigned char marker=0;
	short id=0;
	grib_file* file;
	*err = grib_read_uchar(fh,&marker);
	if(marker == NULL_MARKER) return NULL;
    if(marker != NOT_NULL_MARKER) {*err=GRIB_CORRUPTED_INDEX;return NULL;}

    file=grib_context_malloc(c,sizeof(grib_file));
    file->name=grib_read_string(c,fh,err);
    if (*err) return NULL;

    *err=grib_read_short(fh,&id);
    file->id=id;
    if (*err) return NULL;

    file->next=grib_read_files(c,fh,err);
    if (*err) return NULL;

    return file;
}


int grib_index_write(grib_index* index,const char* filename) {
	int err=0;
	FILE* fh;
	grib_file* files;
	
	fh=fopen(filename,"w");
	if (!fh) {
		grib_context_log(index->context,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
		            "Unable to write in file %s",filename);
		perror(filename);
		return GRIB_IO_PROBLEM;
	}

	err=grib_write_identifier(fh);
	if (err) {
		grib_context_log(index->context,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
			"Unable to write in file %s",filename);
		perror(filename);
		return err;
	}

	if (!index) return grib_write_null_marker(fh);

	err=grib_write_not_null_marker(fh);
	if (err) return err;

	files=grib_file_pool_get_files();
	err=grib_write_files(fh,files);
	if (err) {
		grib_context_log(index->context,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
			"Unable to write in file %s",filename);
		perror(filename);
		return err;
	}


	err=grib_write_index_keys(fh,index->keys);
	if (err) {
		grib_context_log(index->context,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
			"Unable to write in file %s",filename);
		perror(filename);
		return err;
	}

	err=grib_write_field_tree(fh,index->fields);
	if (err) {
		grib_context_log(index->context,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
			"Unable to write in file %s",filename);
		perror(filename);
		return err;
	}

	fclose(fh);
	return err;
}

grib_index* grib_index_read(grib_context* c,const char* filename,int *err) {
	grib_file *file,*f;
	grib_file** files;
	grib_index* index=NULL;
	unsigned char marker=0;
	char* identifier=NULL;
	int max=0;
	FILE* fh;

  if (!c) c=grib_context_get_default();

	fh=fopen(filename,"r");
	if (!fh) {
		grib_context* c = grib_context_get_default();
		grib_context_log(c,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
		            "Unable to write in file %s",filename);
		perror(filename);
		*err=GRIB_IO_PROBLEM;
		return NULL;
	}

	identifier=grib_read_string(c,fh,err);
	if (!identifier) return NULL;
	grib_context_free(c,identifier);

	*err = grib_read_uchar(fh,&marker);
	if(marker == NULL_MARKER) return NULL;
    if(marker != NOT_NULL_MARKER) {*err=GRIB_CORRUPTED_INDEX;return NULL;}

	file = grib_read_files(c,fh,err);
	if (*err) return NULL;

	f=file;
	while (f) {
		if (max<f->id) max=f->id;
		f=f->next;
	}

	files=grib_context_malloc_clear(c,sizeof(grib_file)*(max+1));

	f=file;
	while (f) {
		grib_file_open(f->name,"r",err);
		if (*err) return NULL;
		files[f->id]=grib_get_file(f->name,err);
		f=f->next;
	}

	while (file) {
		f=file;
		file=file->next;
		grib_context_free(c,f->name);
		grib_context_free(c,f);
	}

	index=grib_context_malloc_clear(c,sizeof(grib_index));
	index->context=c;

	index->keys=grib_read_index_keys(c,fh,err);
	if (*err) return NULL;

	index_count=0;
	index->fields=grib_read_field_tree(c,fh,files,err);
	if (*err) return NULL;

	index->count=index_count;

	fclose(fh);
	return index;
}

int grib_index_search_same(grib_index* index,grib_handle* h) {
	int err=0;
	char buf[1024]={0,};
	size_t buflen=1024;
	grib_index_key* keys;
	long lval=0;
	double dval=0.0;
	grib_context* c;

 	if (!index) return GRIB_NULL_INDEX;
	c=index->context;

	keys=index->keys;
		
	while (keys) {
            if (keys->type==GRIB_TYPE_UNDEFINED) {
                err=grib_get_native_type(h,keys->name,&(keys->type));
                if (err) keys->type=GRIB_TYPE_STRING;
            }
            buflen=1024;
            switch (keys->type) {
                case GRIB_TYPE_STRING:
                    err=grib_get_string(h,keys->name,buf,&buflen);
                    if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
                    break;
                case GRIB_TYPE_LONG:
                    err=grib_get_long(h,keys->name,&lval);
                    if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
                    else sprintf(buf,"%ld",lval);
                    break;
                case GRIB_TYPE_DOUBLE:
                    err=grib_get_double(h,keys->name,&dval);
                    if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
                    else sprintf(buf,"%g",dval);
                    break;
                default :
                    err=GRIB_WRONG_TYPE;
                    return err;
            }
            if (err && err != GRIB_NOT_FOUND) {
                grib_context_log(c,GRIB_LOG_ERROR,
					"unable to create index. \"%s\": %s",
					keys->name,grib_get_error_message(err));
                return err;
            }
			sprintf(keys->value,"%s",buf);
			keys=keys->next;
	}
	grib_index_rewind(index);
	return 0;
}

int grib_index_add_file(grib_index* index,const char* filename) {
	double dval;
	size_t svallen;
	long length,lval;
	char buf[1024]={0,};
	int err=0;
	grib_file* indfile;
	grib_file* newfile;
	
	grib_index_key* index_key=NULL;
	grib_handle* h=NULL;
	grib_field* field;
	grib_field_tree* field_tree;
	grib_file* file=NULL;
	grib_context* c;
	
	if (!index) return GRIB_NULL_INDEX;
	c=index->context;
	
	file=grib_file_open(filename,"r",&err);
	if (!file || !file->handle) return err;

	if (!index->files) {
		grib_filesid++;
		newfile=grib_context_malloc_clear(c,sizeof(grib_file));
		newfile->id=grib_filesid;
		newfile->name=strdup(file->name);
		index->files=newfile;
	} else {
		indfile=index->files;
		while(indfile) {
			if (!strcmp(indfile->name,file->name)) return 0;
			indfile=indfile->next;
		}
		indfile=index->files;
		while(indfile->next) indfile=indfile->next;
		grib_filesid++;
		newfile=grib_context_malloc_clear(c,sizeof(grib_file));
		newfile->id=grib_filesid;
		newfile->name=file->name;
		indfile->next=newfile;
	}
    
	fseeko(file->handle,0,SEEK_SET);

	while ((h=grib_handle_new_from_file(c,file->handle,&err))!=NULL) {
		grib_string_list* v=0;
		index_key=index->keys;
		field_tree=index->fields;
		index_key->value[0]=0;

		/* process only GRIB for the moment*/
		svallen=1024;
		grib_get_string(h,"identifier",buf,&svallen);
		if (strcmp(buf,"GRIB")) {
			grib_handle_delete(h);
			return 0;
		}

		while (index_key) {
			if (index_key->type==GRIB_TYPE_UNDEFINED) {
				err=grib_get_native_type(h,index_key->name,&(index_key->type));
				if (err) index_key->type=GRIB_TYPE_STRING;
			}
			svallen=1024;
			switch (index_key->type) {
				case GRIB_TYPE_STRING:
					err=grib_get_string(h,index_key->name,buf,&svallen);
					if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
					break;
				case GRIB_TYPE_LONG:
					err=grib_get_long(h,index_key->name,&lval);
					if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
					else sprintf(buf,"%ld",lval);
					break;
				case GRIB_TYPE_DOUBLE:
					err=grib_get_double(h,index_key->name,&dval);
					if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
					else sprintf(buf,"%g",dval);
					break;
				default :
					err=GRIB_WRONG_TYPE;
					return err;
			}
			if (err && err != GRIB_NOT_FOUND) {
				grib_context_log(c,GRIB_LOG_ERROR,"unable to create index. \"%s\": %s",index_key->name,grib_get_error_message(err));
				return err;
			}

			if (!index_key->values->value) {
				index_key->values->value=grib_context_strdup(c,buf);
				index_key->values_count++;
			} else {
				v=index_key->values;
				while (v->next && strcmp(v->value,buf)) v=v->next;
				if (strcmp(v->value,buf)) {
					index_key->values_count++;
					if (v->next) v=v->next;
					v->next=grib_context_malloc_clear(c,sizeof(grib_string_list));
					v->next->value=grib_context_strdup(c,buf);
				}
			}

			if (!field_tree->value) {
				field_tree->value=grib_context_strdup(c,buf);
			} else {
				while (field_tree->next &&
								   (field_tree->value==NULL ||
								   strcmp(field_tree->value,buf)))
					field_tree=field_tree->next;

				if (!field_tree->value || strcmp(field_tree->value,buf)){
					field_tree->next=
							(grib_field_tree*)grib_context_malloc_clear(c,
							 sizeof(grib_field_tree));
					field_tree=field_tree->next;
					field_tree->value=grib_context_strdup(c,buf);
				}
			}

			if (index_key->next) {
				if (!field_tree->next_level) {
					field_tree->next_level=
							grib_context_malloc_clear(c,sizeof(grib_field_tree));
				}
				field_tree=field_tree->next_level;
			}
			index_key=index_key->next;
		}

		field=grib_context_malloc_clear(c,sizeof(grib_field));
		field->file=file;
		index->count++;
		field->offset=h->offset;;

		err=grib_get_long(h,"totalLength",&length);
		if (err) return err;
		field->length=length;


		if (field_tree->field) {
			grib_field* pfield=field_tree->field;
			while (pfield->next) pfield=pfield->next;
			pfield->next=field;
		} else
			field_tree->field=field;

		if (h) grib_handle_delete(h);

	}

	grib_file_close(file->name,&err);
	if (err) return err;
	index->rewind=1;
	return GRIB_SUCCESS;
	
}

grib_index* grib_index_new_from_file(grib_context* c,
		char* filename,const char* keys,int *err) {
	grib_index* index=NULL;

	if(!c) c=grib_context_get_default();

	index=grib_index_new(c,keys,err);

	*err=grib_index_add_file(index,filename);
	if (*err) {
		grib_index_delete(index);
		return NULL;
	}

	return index;
}

int grib_index_get_size(grib_index* index,const char* key,size_t* size) {
	grib_index_key* k=index->keys;
	while (k && strcmp(k->name,key)) k=k->next;
	if (!k) return GRIB_NOT_FOUND;
	*size=k->values_count;
	return 0;
}

int grib_index_get_string(grib_index* index,const char* key,
		char** values,size_t *size) {
	grib_index_key* k=index->keys;
	grib_string_list* kv;
	int i=0;
	while (k && strcmp(k->name,key)) k=k->next;
	if (!k) return GRIB_NOT_FOUND;
	if (k->values_count>*size) return GRIB_ARRAY_TOO_SMALL;
	kv=k->values;
	while (kv) {
		values[i++]=grib_context_strdup(index->context,kv->value);
		kv=kv->next;
	}
	*size=k->values_count;
    qsort(values,*size,sizeof(char*),&compare_string);
    
	return GRIB_SUCCESS;
}

int grib_index_get_long(grib_index* index,const char* key,
		long* values,size_t *size) {
	grib_index_key* k=index->keys;
	grib_string_list* kv;
	int i=0;
	while (k && strcmp(k->name,key)) k=k->next;
	if (!k) return GRIB_NOT_FOUND;
	if (k->type != GRIB_TYPE_LONG) {
		grib_context_log(index->context,GRIB_LOG_ERROR,
				"unable to get index %s as long");
		return GRIB_WRONG_TYPE;
	}
	if (k->values_count > *size) return GRIB_ARRAY_TOO_SMALL;
	kv=k->values;
	while (kv) {
		if (strcmp(kv->value,GRIB_KEY_UNDEF) )
                  values[i++]=atol(kv->value);
        else
          values[i++]=UNDEF_LONG;
		kv=kv->next;
	}
	*size=k->values_count;
	qsort(values,*size,sizeof(long),&compare_long);

	return GRIB_SUCCESS;
}

int grib_index_get_double(grib_index* index,const char* key,
		double* values,size_t *size) {
	grib_index_key* k=index->keys;
	grib_string_list* kv;
	int i=0;
	while (k && strcmp(k->name,key)) k=k->next;
	if (!k) return GRIB_NOT_FOUND;
	if (k->type != GRIB_TYPE_DOUBLE) {
		grib_context_log(index->context,GRIB_LOG_ERROR,
				"unable to get index %s as double");
		return GRIB_WRONG_TYPE;
	}
	if (k->values_count>*size) return GRIB_ARRAY_TOO_SMALL;
	kv=k->values;
	while (kv) {
      if (strcmp(kv->value,GRIB_KEY_UNDEF) )
		values[i++]=atof(kv->value);
      else
        values[i++]=UNDEF_DOUBLE;
      
		kv=kv->next;
	}
	*size=k->values_count;
	qsort(values,*size,sizeof(double),&compare_double);

	return GRIB_SUCCESS;
}

int grib_index_select_long(grib_index* index,const char* skey,long value) {
	grib_index_key* key=NULL;
	int err=GRIB_NOT_FOUND;

	if (!index) {
		grib_context* c=grib_context_get_default();
		grib_context_log(c,GRIB_LOG_ERROR,"null index pointer");
		return GRIB_INTERNAL_ERROR;
	}
    index->orderby=0;
	key=index->keys;

	while (key) {
		if (!strcmp(key->name,skey)) {
			err=0;
			break;
		}
		key=key->next;
	}

	if (err) {
		grib_context_log(index->context,GRIB_LOG_ERROR,
				"key \"%s\" not found in index",skey);
		return err;
	}

	sprintf(key->value,"%ld",value);
	grib_index_rewind(index);
	return 0;
}

int grib_index_select_double(grib_index* index,const char* skey,double value) {
	grib_index_key* key=NULL;
	int err=GRIB_NOT_FOUND;

	if (!index) {
		grib_context* c=grib_context_get_default();
		grib_context_log(c,GRIB_LOG_ERROR,"null index pointer");
		return GRIB_INTERNAL_ERROR;
	}
    index->orderby=0;
	key=index->keys;

	while (key ) {
		if (!strcmp(key->name,skey)) {
			err=0;
			break;
		}
		key=key->next;
	}

	if (err) {
		grib_context_log(index->context,GRIB_LOG_ERROR,
				"key \"%s\" not found in index",skey);
		return err;
	}

	sprintf(key->value,"%g",value);
	grib_index_rewind(index);
	return 0;
}

int grib_index_select_string(grib_index* index,const char* skey,char* value) {
	grib_index_key* key=NULL;
	int err=GRIB_NOT_FOUND;

	if (!index) {
		grib_context* c=grib_context_get_default();
		grib_context_log(c,GRIB_LOG_ERROR,"null index pointer");
		return GRIB_INTERNAL_ERROR;
	}
    index->orderby=0;
	key=index->keys;

	while (key ) {
		if (!strcmp(key->name,skey)) {
			err=0;
			break;
		}
		key=key->next;
	}

	if (err) {
		grib_context_log(index->context,GRIB_LOG_ERROR,
				"key \"%s\" not found in index",skey);
		return err;
	}

	sprintf(key->value,"%s",value);
	grib_index_rewind(index);
	return 0;
}

grib_handle* grib_index_get_handle(grib_field* field,int *err) {
	grib_handle* h=NULL;
	grib_file_open(field->file->name,"r",err);
	if (*err!=GRIB_SUCCESS) return NULL;

	fseeko(field->file->handle,field->offset,SEEK_SET);
	h=grib_handle_new_from_file(0,field->file->handle,err);
	if (*err!=GRIB_SUCCESS) return NULL;

	grib_file_close(field->file->name,err);
	return h;
}

static int grib_index_execute(grib_index* index) {
	grib_index_key* keys=index->keys;
	grib_field_tree* fields;

	if (!index) return GRIB_INTERNAL_ERROR;

    fields=index->fields;
	index->rewind=0;

	while (keys) {
		char* value;
		if (keys->value[0]) value=keys->value;
		else {
			grib_context_log(index->context,GRIB_LOG_ERROR,
					"please select a value for index key \"%s\"",
					keys->name);
			return GRIB_NOT_FOUND;
		}

		while (fields && strcmp(fields->value,value))
			fields=fields->next;
		if (fields && !strcmp(fields->value,value)) {
			if (fields->next_level) {
				keys=keys->next;
				fields=fields->next_level;
			} else {
				index->current=index->fieldset;
				while(index->current->next) index->current=index->current->next;
				index->current->field=fields->field;
				return 0;
			}
		} else return GRIB_END_OF_INDEX;
	}

	return 0;
}

void grib_index_dump(grib_index* index) {
	
}

char* grib_get_field_file(grib_index* index,off_t *offset) {
	char* file=NULL;
	if (index && index->current && index->current->field) {
		file=index->current->field->file->name;
		*offset=index->current->field->offset;
	}
	return file;
}

grib_handle* grib_handle_new_from_index(grib_index* index,int *err) {
	grib_index_key* keys;
	grib_field_list *fieldset,*next;
	grib_handle* h=NULL;
	grib_context* c=NULL;

	if (!index) return NULL;
	c=index->context;
	if (!index->rewind) {
		if (!index->current) {
                *err=GRIB_END_OF_INDEX;
                return NULL;
            }

		if (index->current->field->next)
			index->current->field=index->current->field->next;
		else if(index->current->next)
			index->current=index->current->next;
		else  {*err=GRIB_END_OF_INDEX;return NULL;}

		h=grib_index_get_handle(index->current->field,err);
		return h;
	}

	if (!index->fieldset) {
		index->fieldset=grib_context_malloc_clear(index->context,
				sizeof(grib_field_list));
		if (!index->fieldset) {
			grib_context_log(index->context,GRIB_LOG_ERROR,
					"unable to allocat %d bytes",
					sizeof(grib_field_list));
			return NULL;
		}
		index->current=index->fieldset;
	} else {
		fieldset=index->fieldset;
		while(fieldset->next) {
			next=fieldset->next;
			grib_context_free(c,fieldset);
			fieldset=next;
		}
		fieldset->field=NULL;
		fieldset->next=NULL;
		index->fieldset=fieldset;
		index->current=fieldset;
	}

	*err=GRIB_END_OF_INDEX;
	h=NULL;
	keys=index->keys;

	if ((*err=grib_index_execute(index))==GRIB_SUCCESS) {

		if (!index->fieldset) {*err=GRIB_END_OF_INDEX;return NULL;}
		index->current=index->fieldset;
		h=grib_index_get_handle(index->current->field,err);
	}
	return h;

}

void grib_index_rewind(grib_index* index) {
	index->rewind=1;
}

static grib_index_key* search_key(grib_index_key* keys,grib_index_key* to_search) {
	if (!keys || !strcmp(keys->name,to_search->name)) return keys;
	return search_key(keys->next,to_search);
}

int grib_index_search(grib_index* index,grib_index_key* keys) {
	
	grib_index_key* ki=index->keys;
	grib_index_key* ks=keys;

	while (ks) {
		ki=search_key(ki,ks);
		if (!ki) {
			ki=index->keys;
			ki=search_key(ki,ks);
		}
		if (ki) sprintf(ki->value,"%s",ks->value);
		ks=ks->next;
	}

	grib_index_rewind(index);
	return 0;
}